2024-11-24 02:41:56,727 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 02:41:56,744 main DEBUG Took 0.014800 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 02:41:56,745 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 02:41:56,746 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 02:41:56,747 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 02:41:56,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,760 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 02:41:56,774 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,776 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,777 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,778 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,780 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,782 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,784 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,786 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,787 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,789 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,790 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,790 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,791 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,791 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:41:56,792 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,792 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 02:41:56,794 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:41:56,796 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 02:41:56,799 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 02:41:56,800 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 02:41:56,802 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 02:41:56,803 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 02:41:56,815 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 02:41:56,819 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 02:41:56,821 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 02:41:56,822 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 02:41:56,823 main DEBUG createAppenders(={Console}) 2024-11-24 02:41:56,824 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 02:41:56,824 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 02:41:56,825 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 02:41:56,825 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 02:41:56,826 main DEBUG OutputStream closed 2024-11-24 02:41:56,826 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 02:41:56,827 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 02:41:56,827 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 02:41:56,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 02:41:56,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 02:41:56,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 02:41:56,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 02:41:56,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 02:41:56,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 02:41:56,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 02:41:56,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 02:41:56,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 02:41:56,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 02:41:56,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 02:41:56,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 02:41:56,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 02:41:56,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 02:41:56,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 02:41:56,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 02:41:56,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 02:41:56,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 02:41:56,940 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 02:41:56,940 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 02:41:56,941 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 02:41:56,942 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T02:41:57,273 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb 2024-11-24 02:41:57,278 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 02:41:57,279 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T02:41:57,293 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-24T02:41:57,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T02:41:57,365 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3, deleteOnExit=true 2024-11-24T02:41:57,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T02:41:57,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/test.cache.data in system properties and HBase conf 2024-11-24T02:41:57,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T02:41:57,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.log.dir in system properties and HBase conf 2024-11-24T02:41:57,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T02:41:57,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T02:41:57,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T02:41:57,493 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T02:41:57,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T02:41:57,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:41:57,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:41:57,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T02:41:57,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:41:57,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T02:41:57,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T02:41:57,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:41:57,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:41:57,635 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T02:41:57,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/nfs.dump.dir in system properties and HBase conf 2024-11-24T02:41:57,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/java.io.tmpdir in system properties and HBase conf 2024-11-24T02:41:57,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:41:57,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T02:41:57,638 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T02:41:58,963 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T02:41:59,035 INFO [Time-limited test {}] log.Log(170): Logging initialized @3034ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T02:41:59,099 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:41:59,151 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:41:59,169 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:41:59,169 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:41:59,170 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:41:59,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:41:59,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:41:59,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:41:59,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/java.io.tmpdir/jetty-localhost-46597-hadoop-hdfs-3_4_1-tests_jar-_-any-2438228357825994087/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:41:59,358 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:46597} 2024-11-24T02:41:59,358 INFO [Time-limited test {}] server.Server(415): Started @3358ms 2024-11-24T02:41:59,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:41:59,929 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:41:59,930 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:41:59,931 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:41:59,931 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:41:59,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:41:59,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:42:00,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/java.io.tmpdir/jetty-localhost-34433-hadoop-hdfs-3_4_1-tests_jar-_-any-16897078863692113870/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:42:00,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:34433} 2024-11-24T02:42:00,032 INFO [Time-limited test {}] server.Server(415): Started @4032ms 2024-11-24T02:42:00,078 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T02:42:01,127 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/data/data2/current/BP-1872727932-172.17.0.2-1732416118302/current, will proceed with Du for space computation calculation, 2024-11-24T02:42:01,127 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/data/data1/current/BP-1872727932-172.17.0.2-1732416118302/current, will proceed with Du for space computation calculation, 2024-11-24T02:42:01,154 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T02:42:01,202 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25fb7cdd0b81c2f with lease ID 0xc1370e6f7baefccc: Processing first storage report for DS-db4894c9-8e5c-4bfe-9af9-c9d392540d79 from datanode DatanodeRegistration(127.0.0.1:46637, datanodeUuid=d0a38681-d74f-4f7b-a7d8-12c8611fc8b0, infoPort=33579, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=1843842749;c=1732416118302) 2024-11-24T02:42:01,203 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25fb7cdd0b81c2f with lease ID 0xc1370e6f7baefccc: from storage DS-db4894c9-8e5c-4bfe-9af9-c9d392540d79 node DatanodeRegistration(127.0.0.1:46637, datanodeUuid=d0a38681-d74f-4f7b-a7d8-12c8611fc8b0, infoPort=33579, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=1843842749;c=1732416118302), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T02:42:01,204 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25fb7cdd0b81c2f with lease ID 0xc1370e6f7baefccc: Processing first storage report for DS-f3f8dd53-972a-432e-8098-c0bde85597fe from datanode DatanodeRegistration(127.0.0.1:46637, datanodeUuid=d0a38681-d74f-4f7b-a7d8-12c8611fc8b0, infoPort=33579, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=1843842749;c=1732416118302) 2024-11-24T02:42:01,204 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25fb7cdd0b81c2f with lease ID 0xc1370e6f7baefccc: from storage DS-f3f8dd53-972a-432e-8098-c0bde85597fe node DatanodeRegistration(127.0.0.1:46637, datanodeUuid=d0a38681-d74f-4f7b-a7d8-12c8611fc8b0, infoPort=33579, infoSecurePort=0, ipcPort=44733, storageInfo=lv=-57;cid=testClusterID;nsid=1843842749;c=1732416118302), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T02:42:01,293 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb 2024-11-24T02:42:01,381 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/zookeeper_0, clientPort=59990, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T02:42:01,392 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59990 2024-11-24T02:42:01,402 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:01,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:01,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741825_1001 (size=7) 2024-11-24T02:42:02,009 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 with version=8 2024-11-24T02:42:02,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/hbase-staging 2024-11-24T02:42:02,087 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T02:42:02,338 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c69a60bd8f6:0 server-side Connection retries=45 2024-11-24T02:42:02,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,347 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,351 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:42:02,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:42:02,471 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T02:42:02,521 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T02:42:02,529 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T02:42:02,532 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:42:02,553 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6870 (auto-detected) 2024-11-24T02:42:02,553 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T02:42:02,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34019 2024-11-24T02:42:02,588 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34019 connecting to ZooKeeper ensemble=127.0.0.1:59990 2024-11-24T02:42:02,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340190x0, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:42:02,745 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34019-0x1016ab825bb0000 connected 2024-11-24T02:42:02,839 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:02,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:02,853 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:42:02,856 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6, hbase.cluster.distributed=false 2024-11-24T02:42:02,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T02:42:02,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34019 2024-11-24T02:42:02,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34019 2024-11-24T02:42:02,885 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34019 2024-11-24T02:42:02,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34019 2024-11-24T02:42:02,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34019 2024-11-24T02:42:02,990 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c69a60bd8f6:0 server-side Connection retries=45 2024-11-24T02:42:02,992 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,992 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,992 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:42:02,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:42:02,993 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:42:02,996 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T02:42:02,999 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:42:02,999 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34251 2024-11-24T02:42:03,002 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34251 connecting to ZooKeeper ensemble=127.0.0.1:59990 2024-11-24T02:42:03,003 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:03,006 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:03,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342510x0, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:42:03,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342510x0, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:42:03,025 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34251-0x1016ab825bb0001 connected 2024-11-24T02:42:03,028 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T02:42:03,035 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T02:42:03,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T02:42:03,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T02:42:03,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-24T02:42:03,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34251 2024-11-24T02:42:03,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34251 2024-11-24T02:42:03,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-24T02:42:03,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-24T02:42:03,059 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c69a60bd8f6:34019 2024-11-24T02:42:03,060 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:03,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:42:03,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:42:03,079 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:03,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T02:42:03,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:03,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:03,111 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T02:42:03,112 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c69a60bd8f6,34019,1732416122189 from backup master directory 2024-11-24T02:42:03,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:03,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:42:03,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:42:03,121 WARN [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:42:03,121 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:03,123 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T02:42:03,125 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T02:42:03,178 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/hbase.id] with ID: a3061a32-fff5-485a-b274-a9772a3f1f74 2024-11-24T02:42:03,178 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/.tmp/hbase.id 2024-11-24T02:42:03,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741826_1002 (size=42) 2024-11-24T02:42:03,590 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/.tmp/hbase.id]:[hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/hbase.id] 2024-11-24T02:42:03,631 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:42:03,636 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T02:42:03,652 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-24T02:42:03,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:03,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:03,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741827_1003 (size=196) 2024-11-24T02:42:04,148 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:42:04,151 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T02:42:04,164 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:04,168 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:42:04,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741828_1004 (size=1189) 2024-11-24T02:42:04,619 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store 2024-11-24T02:42:04,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741829_1005 (size=34) 2024-11-24T02:42:05,050 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T02:42:05,053 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:05,054 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:42:05,054 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:42:05,055 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:42:05,056 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:42:05,056 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:42:05,056 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:42:05,058 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732416125054Disabling compacts and flushes for region at 1732416125054Disabling writes for close at 1732416125056 (+2 ms)Writing region close event to WAL at 1732416125056Closed at 1732416125056 2024-11-24T02:42:05,060 WARN [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/.initializing 2024-11-24T02:42:05,060 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/WALs/7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:05,067 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T02:42:05,080 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C34019%2C1732416122189, suffix=, logDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/WALs/7c69a60bd8f6,34019,1732416122189, archiveDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/oldWALs, maxLogs=10 2024-11-24T02:42:05,099 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/WALs/7c69a60bd8f6,34019,1732416122189/7c69a60bd8f6%2C34019%2C1732416122189.1732416125084, exclude list is [], retry=0 2024-11-24T02:42:05,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46637,DS-db4894c9-8e5c-4bfe-9af9-c9d392540d79,DISK] 2024-11-24T02:42:05,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T02:42:05,147 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/WALs/7c69a60bd8f6,34019,1732416122189/7c69a60bd8f6%2C34019%2C1732416122189.1732416125084 2024-11-24T02:42:05,148 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T02:42:05,149 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:42:05,149 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:05,152 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,153 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T02:42:05,209 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:05,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:05,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T02:42:05,215 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:05,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:05,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T02:42:05,219 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:05,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:05,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T02:42:05,224 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:05,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:05,225 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,229 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,230 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,235 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,236 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,239 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T02:42:05,243 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:42:05,247 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:42:05,249 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59623129, jitterRate=-0.11154614388942719}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T02:42:05,255 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732416125164Initializing all the Stores at 1732416125166 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416125166Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416125167 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416125167Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416125167Cleaning up temporary data from old regions at 1732416125236 (+69 ms)Region opened successfully at 1732416125255 (+19 ms) 2024-11-24T02:42:05,256 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T02:42:05,289 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e91f407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c69a60bd8f6/172.17.0.2:0 2024-11-24T02:42:05,317 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T02:42:05,326 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T02:42:05,327 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T02:42:05,330 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T02:42:05,331 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T02:42:05,335 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T02:42:05,335 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T02:42:05,358 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T02:42:05,366 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T02:42:05,425 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T02:42:05,429 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T02:42:05,433 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T02:42:05,446 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T02:42:05,448 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T02:42:05,451 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T02:42:05,456 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T02:42:05,458 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T02:42:05,467 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T02:42:05,486 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T02:42:05,498 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T02:42:05,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T02:42:05,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T02:42:05,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,513 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c69a60bd8f6,34019,1732416122189, sessionid=0x1016ab825bb0000, setting cluster-up flag (Was=false) 2024-11-24T02:42:05,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,573 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T02:42:05,577 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:05,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:05,636 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T02:42:05,640 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:05,651 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T02:42:05,716 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T02:42:05,724 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T02:42:05,729 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T02:42:05,734 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c69a60bd8f6,34019,1732416122189 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T02:42:05,741 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:42:05,741 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:42:05,741 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:42:05,741 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:42:05,742 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c69a60bd8f6:0, corePoolSize=10, maxPoolSize=10 2024-11-24T02:42:05,742 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,742 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:42:05,742 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,743 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732416155743 2024-11-24T02:42:05,745 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T02:42:05,746 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T02:42:05,747 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:42:05,747 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T02:42:05,749 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T02:42:05,749 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T02:42:05,749 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T02:42:05,749 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T02:42:05,751 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(746): ClusterId : a3061a32-fff5-485a-b274-a9772a3f1f74 2024-11-24T02:42:05,750 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,753 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T02:42:05,754 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:05,754 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T02:42:05,755 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T02:42:05,754 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T02:42:05,755 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T02:42:05,756 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T02:42:05,757 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T02:42:05,759 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416125758,5,FailOnTimeoutGroup] 2024-11-24T02:42:05,760 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416125759,5,FailOnTimeoutGroup] 2024-11-24T02:42:05,760 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,760 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T02:42:05,761 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,762 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,764 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T02:42:05,764 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T02:42:05,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741831_1007 (size=1321) 2024-11-24T02:42:05,774 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T02:42:05,775 DEBUG [RS:0;7c69a60bd8f6:34251 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae8b194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c69a60bd8f6/172.17.0.2:0 2024-11-24T02:42:05,788 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c69a60bd8f6:34251 2024-11-24T02:42:05,790 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T02:42:05,791 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T02:42:05,791 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T02:42:05,793 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c69a60bd8f6,34019,1732416122189 with port=34251, startcode=1732416122955 2024-11-24T02:42:05,802 DEBUG [RS:0;7c69a60bd8f6:34251 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T02:42:05,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T02:42:05,843 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34019 {}] master.ServerManager(517): Registering regionserver=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,861 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:42:05,861 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41729 2024-11-24T02:42:05,861 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T02:42:05,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T02:42:05,877 DEBUG [RS:0;7c69a60bd8f6:34251 {}] zookeeper.ZKUtil(111): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,877 WARN [RS:0;7c69a60bd8f6:34251 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:42:05,877 INFO [RS:0;7c69a60bd8f6:34251 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:42:05,877 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,879 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c69a60bd8f6,34251,1732416122955] 2024-11-24T02:42:05,902 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T02:42:05,917 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T02:42:05,922 INFO [RS:0;7c69a60bd8f6:34251 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T02:42:05,922 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,923 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T02:42:05,927 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T02:42:05,929 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,929 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,929 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,929 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,929 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,929 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c69a60bd8f6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,930 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:42:05,931 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c69a60bd8f6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:42:05,931 DEBUG [RS:0;7c69a60bd8f6:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:42:05,931 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,932 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,932 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,932 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,932 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,932 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34251,1732416122955-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:42:05,950 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T02:42:05,952 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34251,1732416122955-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,952 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,952 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.Replication(171): 7c69a60bd8f6,34251,1732416122955 started 2024-11-24T02:42:05,969 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:05,969 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1482): Serving as 7c69a60bd8f6,34251,1732416122955, RpcServer on 7c69a60bd8f6/172.17.0.2:34251, sessionid=0x1016ab825bb0001 2024-11-24T02:42:05,970 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T02:42:05,970 DEBUG [RS:0;7c69a60bd8f6:34251 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,970 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c69a60bd8f6,34251,1732416122955' 2024-11-24T02:42:05,970 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T02:42:05,971 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T02:42:05,972 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T02:42:05,972 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T02:42:05,972 DEBUG [RS:0;7c69a60bd8f6:34251 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:05,972 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c69a60bd8f6,34251,1732416122955' 2024-11-24T02:42:05,972 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T02:42:05,973 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T02:42:05,973 DEBUG [RS:0;7c69a60bd8f6:34251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T02:42:05,974 INFO [RS:0;7c69a60bd8f6:34251 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T02:42:05,974 INFO [RS:0;7c69a60bd8f6:34251 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T02:42:06,080 INFO [RS:0;7c69a60bd8f6:34251 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T02:42:06,085 INFO [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C34251%2C1732416122955, suffix=, logDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955, archiveDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/oldWALs, maxLogs=32 2024-11-24T02:42:06,101 DEBUG [RS:0;7c69a60bd8f6:34251 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955/7c69a60bd8f6%2C34251%2C1732416122955.1732416126088, exclude list is [], retry=0 2024-11-24T02:42:06,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46637,DS-db4894c9-8e5c-4bfe-9af9-c9d392540d79,DISK] 2024-11-24T02:42:06,109 INFO [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955/7c69a60bd8f6%2C34251%2C1732416122955.1732416126088 2024-11-24T02:42:06,110 DEBUG [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T02:42:06,170 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T02:42:06,171 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:42:06,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741833_1009 (size=32) 2024-11-24T02:42:06,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:06,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:42:06,596 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:42:06,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:06,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:06,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:42:06,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:42:06,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:06,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:06,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:42:06,603 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:42:06,603 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:06,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:06,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:42:06,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:42:06,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:06,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:06,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:42:06,609 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740 2024-11-24T02:42:06,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740 2024-11-24T02:42:06,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:42:06,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:42:06,614 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:42:06,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:42:06,619 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:42:06,620 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60572396, jitterRate=-0.097400963306427}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:42:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732416126588Initializing all the Stores at 1732416126591 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416126591Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416126592 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416126592Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416126592Cleaning up temporary data from old regions at 1732416126613 (+21 ms)Region opened successfully at 1732416126623 (+10 ms) 2024-11-24T02:42:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:42:06,623 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:42:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:42:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:42:06,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:42:06,624 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:42:06,625 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732416126623Disabling compacts and flushes for region at 1732416126623Disabling writes for close at 1732416126623Writing region close event to WAL at 1732416126624 (+1 ms)Closed at 1732416126624 2024-11-24T02:42:06,628 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:42:06,628 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T02:42:06,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T02:42:06,640 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:42:06,643 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T02:42:06,798 DEBUG [7c69a60bd8f6:34019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T02:42:06,813 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:06,819 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c69a60bd8f6,34251,1732416122955, state=OPENING 2024-11-24T02:42:06,867 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T02:42:06,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:06,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:42:06,879 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:42:06,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:42:06,882 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:42:06,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:07,066 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T02:42:07,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T02:42:07,080 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T02:42:07,080 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:42:07,081 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T02:42:07,084 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C34251%2C1732416122955.meta, suffix=.meta, logDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955, archiveDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/oldWALs, maxLogs=32 2024-11-24T02:42:07,100 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955/7c69a60bd8f6%2C34251%2C1732416122955.meta.1732416127086.meta, exclude list is [], retry=0 2024-11-24T02:42:07,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46637,DS-db4894c9-8e5c-4bfe-9af9-c9d392540d79,DISK] 2024-11-24T02:42:07,107 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/WALs/7c69a60bd8f6,34251,1732416122955/7c69a60bd8f6%2C34251%2C1732416122955.meta.1732416127086.meta 2024-11-24T02:42:07,107 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33579:33579)] 2024-11-24T02:42:07,107 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:42:07,109 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T02:42:07,112 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T02:42:07,116 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T02:42:07,120 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T02:42:07,121 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:07,121 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T02:42:07,121 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T02:42:07,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:42:07,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:42:07,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:07,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:07,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:42:07,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:42:07,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:07,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:07,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:42:07,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:42:07,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:07,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:07,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:42:07,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:42:07,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:07,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:42:07,135 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:42:07,136 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740 2024-11-24T02:42:07,139 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740 2024-11-24T02:42:07,141 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:42:07,141 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:42:07,142 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:42:07,145 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:42:07,146 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70384488, jitterRate=0.048810601234436035}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:42:07,147 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T02:42:07,148 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732416127121Writing region info on filesystem at 1732416127122 (+1 ms)Initializing all the Stores at 1732416127123 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416127123Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416127124 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416127124Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416127124Cleaning up temporary data from old regions at 1732416127141 (+17 ms)Running coprocessor post-open hooks at 1732416127147 (+6 ms)Region opened successfully at 1732416127148 (+1 ms) 2024-11-24T02:42:07,154 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732416127057 2024-11-24T02:42:07,164 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T02:42:07,165 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T02:42:07,166 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:07,168 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c69a60bd8f6,34251,1732416122955, state=OPEN 2024-11-24T02:42:07,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T02:42:07,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T02:42:07,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:42:07,212 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:42:07,212 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:07,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T02:42:07,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,34251,1732416122955 in 327 msec 2024-11-24T02:42:07,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T02:42:07,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 588 msec 2024-11-24T02:42:07,229 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:42:07,229 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T02:42:07,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:07,248 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:07,271 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:07,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59927, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:07,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6130 sec 2024-11-24T02:42:07,292 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732416127292, completionTime=-1 2024-11-24T02:42:07,295 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T02:42:07,295 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T02:42:07,319 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T02:42:07,319 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732416187319 2024-11-24T02:42:07,319 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732416247319 2024-11-24T02:42:07,319 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-24T02:42:07,321 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,322 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,322 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,323 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c69a60bd8f6:34019, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,323 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,324 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,330 DEBUG [master/7c69a60bd8f6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T02:42:07,353 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.231sec 2024-11-24T02:42:07,354 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T02:42:07,355 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T02:42:07,356 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T02:42:07,357 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T02:42:07,357 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T02:42:07,358 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:42:07,358 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T02:42:07,365 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T02:42:07,366 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T02:42:07,367 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,34019,1732416122189-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:42:07,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:07,486 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T02:42:07,486 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T02:42:07,489 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:07,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:07,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6517db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:07,511 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:07,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:07,515 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:07,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:07,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:07,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=237, ProcessCount=11, AvailableMemoryMB=10548 2024-11-24T02:42:07,563 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:42:07,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:07,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23412a84 2024-11-24T02:42:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:42:07,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:42:07,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:42:07,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:42:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:07,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:42:07,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T02:42:07,597 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:07,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:42:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:07,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741835_1011 (size=1061) 2024-11-24T02:42:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:08,018 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:42:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741836_1012 (size=53) 2024-11-24T02:42:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:08,431 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:08,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94fe1d236a5fa4edae1b3d10d452f90e, disabling compactions & flushes 2024-11-24T02:42:08,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. after waiting 0 ms 2024-11-24T02:42:08,432 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,433 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,433 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94fe1d236a5fa4edae1b3d10d452f90e: Waiting for close lock at 1732416128432Disabling compacts and flushes for region at 1732416128432Disabling writes for close at 1732416128432Writing region close event to WAL at 1732416128433 (+1 ms)Closed at 1732416128433 2024-11-24T02:42:08,437 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:42:08,445 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416128437"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416128437"}]},"ts":"1732416128437"} 2024-11-24T02:42:08,449 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:42:08,451 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:42:08,454 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416128451"}]},"ts":"1732416128451"} 2024-11-24T02:42:08,458 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:42:08,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, ASSIGN}] 2024-11-24T02:42:08,463 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, ASSIGN 2024-11-24T02:42:08,465 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:42:08,617 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94fe1d236a5fa4edae1b3d10d452f90e, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:08,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, ASSIGN because future has completed 2024-11-24T02:42:08,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:08,793 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,793 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:42:08,794 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,794 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:08,794 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,795 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,798 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,803 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:08,804 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94fe1d236a5fa4edae1b3d10d452f90e columnFamilyName A 2024-11-24T02:42:08,804 DEBUG [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:08,806 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(327): Store=94fe1d236a5fa4edae1b3d10d452f90e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:08,806 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,809 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:08,809 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94fe1d236a5fa4edae1b3d10d452f90e columnFamilyName B 2024-11-24T02:42:08,809 DEBUG [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:08,810 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(327): Store=94fe1d236a5fa4edae1b3d10d452f90e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:08,811 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,813 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:08,814 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94fe1d236a5fa4edae1b3d10d452f90e columnFamilyName C 2024-11-24T02:42:08,814 DEBUG [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:08,815 INFO [StoreOpener-94fe1d236a5fa4edae1b3d10d452f90e-1 {}] regionserver.HStore(327): Store=94fe1d236a5fa4edae1b3d10d452f90e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:08,816 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,816 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,818 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,820 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,822 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,823 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,824 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:42:08,827 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,832 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:42:08,836 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 94fe1d236a5fa4edae1b3d10d452f90e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69373486, jitterRate=0.03374549746513367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:42:08,836 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:08,838 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 94fe1d236a5fa4edae1b3d10d452f90e: Running coprocessor pre-open hook at 1732416128795Writing region info on filesystem at 1732416128795Initializing all the Stores at 1732416128797 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416128797Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416128798 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416128798Cleaning up temporary data from old regions at 1732416128823 (+25 ms)Running coprocessor post-open hooks at 1732416128837 (+14 ms)Region opened successfully at 1732416128838 (+1 ms) 2024-11-24T02:42:08,841 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., pid=6, masterSystemTime=1732416128784 2024-11-24T02:42:08,846 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,846 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:08,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94fe1d236a5fa4edae1b3d10d452f90e, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:08,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:42:08,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T02:42:08,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 in 226 msec 2024-11-24T02:42:08,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T02:42:08,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, ASSIGN in 398 msec 2024-11-24T02:42:08,865 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:42:08,865 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416128865"}]},"ts":"1732416128865"} 2024-11-24T02:42:08,869 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:42:08,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:42:08,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2860 sec 2024-11-24T02:42:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:42:09,751 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:09,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c078f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,760 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c107cb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,768 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:42:09,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a200d59 2024-11-24T02:42:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:42:09,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:42:09,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ccbe83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ac8568, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,788 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f535689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,796 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467a1192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c468bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c220cf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,811 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140d06c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,813 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a045d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ef4082b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bfb1d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253eb29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f76377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf48d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a4abfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:09,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:09,837 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbd72e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:09,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:09,840 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3edeb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:09,845 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,845 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,845 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,845 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,848 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,849 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,849 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,854 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,857 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T02:42:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,861 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,866 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:09,872 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:09,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:09,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:42:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:09,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:42:09,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:09,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:10,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/5cfec8a97a24495ca9541b594c94ec23 is 50, key is test_row_0/A:col10/1732416129943/Put/seqid=0 2024-11-24T02:42:10,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741837_1013 (size=12001) 2024-11-24T02:42:10,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/5cfec8a97a24495ca9541b594c94ec23 2024-11-24T02:42:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416130074, max timestamp=1732416130143Num uniques 859; 2024-11-24T02:42:10,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732416130074, max timestamp=1732416130143Num uniques 859; 2024-11-24T02:42:10,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416190101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416130074, max timestamp=1732416130151Num uniques 859; 2024-11-24T02:42:10,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:10,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:10,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/26d94f61215348d8ab3531f4b280554f is 50, key is test_row_0/B:col10/1732416129943/Put/seqid=0 2024-11-24T02:42:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741838_1014 (size=12001) 2024-11-24T02:42:10,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416190248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416190254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416190257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416190260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416190320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416190371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416190371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416190381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416190381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:10,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416190539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416190580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416190591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416190591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416190591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:10,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/26d94f61215348d8ab3531f4b280554f 2024-11-24T02:42:10,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/daf87161e77448c1945d1466bcbb30f5 is 50, key is test_row_0/C:col10/1732416129943/Put/seqid=0 2024-11-24T02:42:10,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741839_1015 (size=12001) 2024-11-24T02:42:10,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/daf87161e77448c1945d1466bcbb30f5 2024-11-24T02:42:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/5cfec8a97a24495ca9541b594c94ec23 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23 2024-11-24T02:42:10,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:42:10,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/26d94f61215348d8ab3531f4b280554f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f 2024-11-24T02:42:10,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:42:10,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/daf87161e77448c1945d1466bcbb30f5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5 2024-11-24T02:42:10,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:42:10,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 94fe1d236a5fa4edae1b3d10d452f90e in 837ms, sequenceid=13, compaction requested=false 2024-11-24T02:42:10,786 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:42:10,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:10,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T02:42:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:10,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:10,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:10,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:10,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:10,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:10,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/76fd934466ac40798071e9d446ff0689 is 50, key is test_row_0/A:col10/1732416130870/Put/seqid=0 2024-11-24T02:42:10,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:10,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:10,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:10,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741840_1016 (size=14341) 2024-11-24T02:42:10,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/76fd934466ac40798071e9d446ff0689 2024-11-24T02:42:10,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/86c2b77ffc9440848fa1f392bb6164bc is 50, key is test_row_0/B:col10/1732416130870/Put/seqid=0 2024-11-24T02:42:10,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741841_1017 (size=12001) 2024-11-24T02:42:10,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/86c2b77ffc9440848fa1f392bb6164bc 2024-11-24T02:42:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416190933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:11,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5317eca6bef74c5dbfa7e31e8a6a20a9 is 50, key is test_row_0/C:col10/1732416130870/Put/seqid=0 2024-11-24T02:42:11,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416190970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416190971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416190979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416190982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741842_1018 (size=12001) 2024-11-24T02:42:11,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5317eca6bef74c5dbfa7e31e8a6a20a9 2024-11-24T02:42:11,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/76fd934466ac40798071e9d446ff0689 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689 2024-11-24T02:42:11,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689, entries=200, sequenceid=48, filesize=14.0 K 2024-11-24T02:42:11,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/86c2b77ffc9440848fa1f392bb6164bc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc 2024-11-24T02:42:11,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T02:42:11,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5317eca6bef74c5dbfa7e31e8a6a20a9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9 2024-11-24T02:42:11,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416191092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T02:42:11,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 94fe1d236a5fa4edae1b3d10d452f90e in 233ms, sequenceid=48, compaction requested=false 2024-11-24T02:42:11,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:11,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:11,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:11,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:11,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cfe788e3148942f0902025d803550d45 is 50, key is test_row_0/A:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741843_1019 (size=14341) 2024-11-24T02:42:11,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cfe788e3148942f0902025d803550d45 2024-11-24T02:42:11,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9ce16832909a4dbdb18db44be4f548c2 is 50, key is test_row_0/B:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741844_1020 (size=12001) 2024-11-24T02:42:11,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416131214, max timestamp=1732416131261Num uniques 859; 2024-11-24T02:42:11,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:11,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416131214, max timestamp=1732416131261Num uniques 859; 2024-11-24T02:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:11,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416131214, max timestamp=1732416131261Num uniques 859; 2024-11-24T02:42:11,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:11,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416191310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416191324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416191328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416191332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416191334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416191440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416191441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416191450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416191450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9ce16832909a4dbdb18db44be4f548c2 2024-11-24T02:42:11,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/021640de4e0a47619be8cdc5905d30e0 is 50, key is test_row_0/C:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741845_1021 (size=12001) 2024-11-24T02:42:11,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/021640de4e0a47619be8cdc5905d30e0 2024-11-24T02:42:11,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416191650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416191650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416191660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cfe788e3148942f0902025d803550d45 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45 2024-11-24T02:42:11,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416191670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416191670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:11,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45, entries=200, sequenceid=60, filesize=14.0 K 2024-11-24T02:42:11,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9ce16832909a4dbdb18db44be4f548c2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2 2024-11-24T02:42:11,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:11,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T02:42:11,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/021640de4e0a47619be8cdc5905d30e0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0 2024-11-24T02:42:11,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T02:42:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 94fe1d236a5fa4edae1b3d10d452f90e in 596ms, sequenceid=60, compaction requested=true 2024-11-24T02:42:11,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:11,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:11,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:11,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:11,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:11,741 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:11,742 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:11,744 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:11,744 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:11,744 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:11,745 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,745 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.2 K 2024-11-24T02:42:11,746 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,746 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=39.7 K 2024-11-24T02:42:11,748 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d94f61215348d8ab3531f4b280554f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416129922 2024-11-24T02:42:11,748 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfec8a97a24495ca9541b594c94ec23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416129922 2024-11-24T02:42:11,749 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c2b77ffc9440848fa1f392bb6164bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416130253 2024-11-24T02:42:11,749 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76fd934466ac40798071e9d446ff0689, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416130241 2024-11-24T02:42:11,750 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce16832909a4dbdb18db44be4f548c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:11,750 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfe788e3148942f0902025d803550d45, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:11,806 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#10 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:11,807 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a23fc494d7114a97980630fbc36306b7 is 50, key is test_row_0/B:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,810 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:11,811 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/71ec500084d041be93484e0ed0121730 is 50, key is test_row_0/A:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741846_1022 (size=12104) 2024-11-24T02:42:11,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:42:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-24T02:42:11,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:11,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:11,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:11,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:11,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:11,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:11,865 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a23fc494d7114a97980630fbc36306b7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a23fc494d7114a97980630fbc36306b7 2024-11-24T02:42:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741847_1023 (size=12104) 2024-11-24T02:42:11,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/f389e6bbaa954bcd8f1133fd7c693e81 is 50, key is test_row_0/A:col10/1732416131327/Put/seqid=0 2024-11-24T02:42:11,891 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into a23fc494d7114a97980630fbc36306b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:11,891 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:11,896 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416131734; duration=0sec 2024-11-24T02:42:11,897 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:11,897 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:11,897 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T02:42:11,898 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:11,902 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:11,902 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:11,902 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:11,903 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.2 K 2024-11-24T02:42:11,904 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting daf87161e77448c1945d1466bcbb30f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416129922 2024-11-24T02:42:11,904 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5317eca6bef74c5dbfa7e31e8a6a20a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416130253 2024-11-24T02:42:11,906 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 021640de4e0a47619be8cdc5905d30e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:11,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741848_1024 (size=12001) 2024-11-24T02:42:11,934 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:11,935 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b973a73992bf4732a04bc26f7349fb0a is 50, key is test_row_0/C:col10/1732416130963/Put/seqid=0 2024-11-24T02:42:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741849_1025 (size=12104) 2024-11-24T02:42:11,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:11,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:11,971 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b973a73992bf4732a04bc26f7349fb0a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b973a73992bf4732a04bc26f7349fb0a 2024-11-24T02:42:11,986 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into b973a73992bf4732a04bc26f7349fb0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:11,986 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:11,986 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416131734; duration=0sec 2024-11-24T02:42:11,986 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:11,986 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:12,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416191996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416191999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416192007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416192120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416192121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416192129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416192169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416192330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416192330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/f389e6bbaa954bcd8f1133fd7c693e81 2024-11-24T02:42:12,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416192342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,352 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/71ec500084d041be93484e0ed0121730 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/71ec500084d041be93484e0ed0121730 2024-11-24T02:42:12,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/409dfe3c433d41ba95fd20d67d039fe4 is 50, key is test_row_0/B:col10/1732416131327/Put/seqid=0 2024-11-24T02:42:12,367 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 71ec500084d041be93484e0ed0121730(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:12,368 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:12,368 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416131731; duration=0sec 2024-11-24T02:42:12,368 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:12,368 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:12,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741850_1026 (size=12001) 2024-11-24T02:42:12,375 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/409dfe3c433d41ba95fd20d67d039fe4 2024-11-24T02:42:12,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5108a5df916f4008a672a535f29b50dd is 50, key is test_row_0/C:col10/1732416131327/Put/seqid=0 2024-11-24T02:42:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741851_1027 (size=12001) 2024-11-24T02:42:12,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5108a5df916f4008a672a535f29b50dd 2024-11-24T02:42:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/f389e6bbaa954bcd8f1133fd7c693e81 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81 2024-11-24T02:42:12,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:42:12,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/409dfe3c433d41ba95fd20d67d039fe4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4 2024-11-24T02:42:12,466 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:42:12,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5108a5df916f4008a672a535f29b50dd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd 2024-11-24T02:42:12,485 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:42:12,487 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 94fe1d236a5fa4edae1b3d10d452f90e in 641ms, sequenceid=94, compaction requested=false 2024-11-24T02:42:12,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:12,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:12,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T02:42:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T02:42:12,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T02:42:12,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6220 sec 2024-11-24T02:42:12,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.6460 sec 2024-11-24T02:42:12,636 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:42:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:12,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:12,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:12,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6e3b97bdfb724d13b2a8de8bc64b5065 is 50, key is test_row_1/A:col10/1732416132644/Put/seqid=0 2024-11-24T02:42:12,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741852_1028 (size=11997) 2024-11-24T02:42:12,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6e3b97bdfb724d13b2a8de8bc64b5065 2024-11-24T02:42:12,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T02:42:12,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T02:42:12,711 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:42:12,711 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T02:42:12,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T02:42:12,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T02:42:12,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:42:12,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T02:42:12,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/90a5652e5b3a41678fb2876462b27b89 is 50, key is test_row_1/B:col10/1732416132644/Put/seqid=0 2024-11-24T02:42:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741853_1029 (size=9657) 2024-11-24T02:42:12,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/90a5652e5b3a41678fb2876462b27b89 2024-11-24T02:42:12,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416192729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416192739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=182, cellSize=8332, totalHeapSize=19168, min timestamp=1732416132714, max timestamp=1732416132732Num uniques 859; 2024-11-24T02:42:12,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8332, suffix heap size=117064,new segment heap size=19168 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=182 2024-11-24T02:42:12,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416132725, max timestamp=1732416132732Num uniques 859; 2024-11-24T02:42:12,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:12,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416132725, max timestamp=1732416132732Num uniques 859; 2024-11-24T02:42:12,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:12,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/da365121aa854463b163da0dd8337b01 is 50, key is test_row_1/C:col10/1732416132644/Put/seqid=0 2024-11-24T02:42:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741854_1030 (size=9657) 2024-11-24T02:42:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416192854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416192854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416192859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416192970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416192970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416192971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:12,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416193069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416193179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416193180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416193180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416193189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,194 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in 94fe1d236a5fa4edae1b3d10d452f90e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/da365121aa854463b163da0dd8337b01 2024-11-24T02:42:13,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6e3b97bdfb724d13b2a8de8bc64b5065 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065 2024-11-24T02:42:13,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065, entries=150, sequenceid=109, filesize=11.7 K 2024-11-24T02:42:13,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/90a5652e5b3a41678fb2876462b27b89 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89 2024-11-24T02:42:13,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89, entries=100, sequenceid=109, filesize=9.4 K 2024-11-24T02:42:13,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/da365121aa854463b163da0dd8337b01 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01 2024-11-24T02:42:13,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01, entries=100, sequenceid=109, filesize=9.4 K 2024-11-24T02:42:13,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=121.79 KB/124718 for 94fe1d236a5fa4edae1b3d10d452f90e in 650ms, sequenceid=109, compaction requested=true 2024-11-24T02:42:13,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,298 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:13,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:13,300 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:13,300 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:13,300 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:13,300 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:13,301 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/71ec500084d041be93484e0ed0121730, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.3 K 2024-11-24T02:42:13,302 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71ec500084d041be93484e0ed0121730, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:13,302 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:13,302 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:13,303 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:13,303 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f389e6bbaa954bcd8f1133fd7c693e81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416131282 2024-11-24T02:42:13,303 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a23fc494d7114a97980630fbc36306b7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=33.0 K 2024-11-24T02:42:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a23fc494d7114a97980630fbc36306b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:13,306 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3b97bdfb724d13b2a8de8bc64b5065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732416131998 2024-11-24T02:42:13,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:13,308 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 409dfe3c433d41ba95fd20d67d039fe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416131282 2024-11-24T02:42:13,311 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a5652e5b3a41678fb2876462b27b89, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732416131998 2024-11-24T02:42:13,332 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:13,333 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/8f1e2bbe41b549f0846cdca4cc4c5fc4 is 50, key is test_row_0/A:col10/1732416131327/Put/seqid=0 2024-11-24T02:42:13,335 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:13,336 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/5c5cda0a1c4c48debe2ac9f2759e4143 is 50, key is test_row_0/B:col10/1732416131327/Put/seqid=0 2024-11-24T02:42:13,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741856_1032 (size=12207) 2024-11-24T02:42:13,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741855_1031 (size=12207) 2024-11-24T02:42:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:13,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=128.50 KB heapSize=367.40 KB 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cedbd4c919b44a37b64b9952770ca95d is 50, key is test_row_0/A:col10/1732416132854/Put/seqid=0 2024-11-24T02:42:13,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741857_1033 (size=14541) 2024-11-24T02:42:13,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.52 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cedbd4c919b44a37b64b9952770ca95d 2024-11-24T02:42:13,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a36a3113da484704851fbbb3821ef334 is 50, key is test_row_0/B:col10/1732416132854/Put/seqid=0 2024-11-24T02:42:13,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741858_1034 (size=12151) 2024-11-24T02:42:13,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a36a3113da484704851fbbb3821ef334 2024-11-24T02:42:13,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416193482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/fa59447681014f9c9b10cd1a7f24e41b is 50, key is test_row_0/C:col10/1732416132854/Put/seqid=0 2024-11-24T02:42:13,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416193500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416193500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416193501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741859_1035 (size=12151) 2024-11-24T02:42:13,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/fa59447681014f9c9b10cd1a7f24e41b 2024-11-24T02:42:13,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cedbd4c919b44a37b64b9952770ca95d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d 2024-11-24T02:42:13,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d, entries=200, sequenceid=145, filesize=14.2 K 2024-11-24T02:42:13,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a36a3113da484704851fbbb3821ef334 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334 2024-11-24T02:42:13,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334, entries=150, sequenceid=145, filesize=11.9 K 2024-11-24T02:42:13,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/fa59447681014f9c9b10cd1a7f24e41b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b 2024-11-24T02:42:13,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b, entries=150, sequenceid=145, filesize=11.9 K 2024-11-24T02:42:13,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~128.50 KB/131588, heapSize ~367.35 KB/376168, currentSize=53.67 KB/54960 for 94fe1d236a5fa4edae1b3d10d452f90e in 198ms, sequenceid=145, compaction requested=true 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T02:42:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:13,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:13,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:13,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d1e7eef88a2b4a31a6514c7a6d3514ea is 50, key is test_row_0/A:col10/1732416133467/Put/seqid=0 2024-11-24T02:42:13,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741860_1036 (size=14541) 2024-11-24T02:42:13,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d1e7eef88a2b4a31a6514c7a6d3514ea 2024-11-24T02:42:13,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/e784256f5d2a4daeb656ce9ebe1c5d73 is 50, key is test_row_0/B:col10/1732416133467/Put/seqid=0 2024-11-24T02:42:13,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741861_1037 (size=12151) 2024-11-24T02:42:13,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/e784256f5d2a4daeb656ce9ebe1c5d73 2024-11-24T02:42:13,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d46724c148ff4f2489af204f19de97d3 is 50, key is test_row_0/C:col10/1732416133467/Put/seqid=0 2024-11-24T02:42:13,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741862_1038 (size=12151) 2024-11-24T02:42:13,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d46724c148ff4f2489af204f19de97d3 2024-11-24T02:42:13,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d1e7eef88a2b4a31a6514c7a6d3514ea as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea 2024-11-24T02:42:13,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea, entries=200, sequenceid=157, filesize=14.2 K 2024-11-24T02:42:13,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/e784256f5d2a4daeb656ce9ebe1c5d73 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73 2024-11-24T02:42:13,768 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/5c5cda0a1c4c48debe2ac9f2759e4143 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5c5cda0a1c4c48debe2ac9f2759e4143 2024-11-24T02:42:13,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416193764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:13,784 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/8f1e2bbe41b549f0846cdca4cc4c5fc4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/8f1e2bbe41b549f0846cdca4cc4c5fc4 2024-11-24T02:42:13,790 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 5c5cda0a1c4c48debe2ac9f2759e4143(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:13,790 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,790 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416133300; duration=0sec 2024-11-24T02:42:13,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T02:42:13,793 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T02:42:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d46724c148ff4f2489af204f19de97d3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3 2024-11-24T02:42:13,793 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:13,794 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-24T02:42:13,795 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:13,796 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:13,796 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:13,796 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:13,796 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:13,800 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:13,801 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:13,801 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:13,802 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b973a73992bf4732a04bc26f7349fb0a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=44.8 K 2024-11-24T02:42:13,803 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b973a73992bf4732a04bc26f7349fb0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732416130963 2024-11-24T02:42:13,804 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5108a5df916f4008a672a535f29b50dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416131282 2024-11-24T02:42:13,805 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting da365121aa854463b163da0dd8337b01, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732416131998 2024-11-24T02:42:13,806 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fa59447681014f9c9b10cd1a7f24e41b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732416132820 2024-11-24T02:42:13,811 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 8f1e2bbe41b549f0846cdca4cc4c5fc4(size=11.9 K), total size for store is 40.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:13,811 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,811 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416133298; duration=0sec 2024-11-24T02:42:13,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T02:42:13,813 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:42:13,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94fe1d236a5fa4edae1b3d10d452f90e in 204ms, sequenceid=157, compaction requested=true 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,814 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:13,814 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T02:42:13,814 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:13,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-24T02:42:13,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:13,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:13,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:13,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:13,817 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-24T02:42:13,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:13,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:13,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:13,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:13,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:13,820 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:13,820 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:13,821 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:13,821 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/8f1e2bbe41b549f0846cdca4cc4c5fc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=40.3 K 2024-11-24T02:42:13,823 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1e2bbe41b549f0846cdca4cc4c5fc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732416131327 2024-11-24T02:42:13,824 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cedbd4c919b44a37b64b9952770ca95d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732416132820 2024-11-24T02:42:13,825 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e7eef88a2b4a31a6514c7a6d3514ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416133458 2024-11-24T02:42:13,831 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:13,832 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/c5c94bc34d86417da1103d2c37014aaf is 50, key is test_row_0/C:col10/1732416132854/Put/seqid=0 2024-11-24T02:42:13,852 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:13,853 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c039fb84848b4cbe98e64a868af82c6b is 50, key is test_row_0/A:col10/1732416133467/Put/seqid=0 2024-11-24T02:42:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741863_1039 (size=12391) 2024-11-24T02:42:13,870 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/c5c94bc34d86417da1103d2c37014aaf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c5c94bc34d86417da1103d2c37014aaf 2024-11-24T02:42:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741864_1040 (size=12459) 2024-11-24T02:42:13,882 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c039fb84848b4cbe98e64a868af82c6b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c039fb84848b4cbe98e64a868af82c6b 2024-11-24T02:42:13,894 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into c5c94bc34d86417da1103d2c37014aaf(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:13,894 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:13,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T02:42:13,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416133733, max timestamp=1732416133893Num uniques 859; 2024-11-24T02:42:13,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:13,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416133733, max timestamp=1732416133893Num uniques 859; 2024-11-24T02:42:13,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,898 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into c039fb84848b4cbe98e64a868af82c6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:13,898 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,898 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416133814; duration=0sec 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:13,898 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=12, startTime=1732416133582; duration=0sec 2024-11-24T02:42:13,898 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:42:13,898 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:13,898 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:13,899 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:13,901 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 3 compacting, 0 eligible, 16 blocking 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:13,901 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:13,902 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:13,902 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:13,902 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5c5cda0a1c4c48debe2ac9f2759e4143, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.7 K 2024-11-24T02:42:13,903 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5cda0a1c4c48debe2ac9f2759e4143, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732416131327 2024-11-24T02:42:13,904 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a36a3113da484704851fbbb3821ef334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732416132820 2024-11-24T02:42:13,909 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting e784256f5d2a4daeb656ce9ebe1c5d73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416133458 2024-11-24T02:42:13,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d2723a62f67643e28ce8cdfe9f1b2544 is 50, key is test_row_0/A:col10/1732416133758/Put/seqid=0 2024-11-24T02:42:13,936 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:13,937 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/6c5287f675d940648d516441f0a0ef91 is 50, key is test_row_0/B:col10/1732416133467/Put/seqid=0 2024-11-24T02:42:13,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741865_1041 (size=12151) 2024-11-24T02:42:13,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741866_1042 (size=12459) 2024-11-24T02:42:13,975 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/6c5287f675d940648d516441f0a0ef91 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/6c5287f675d940648d516441f0a0ef91 2024-11-24T02:42:13,997 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 6c5287f675d940648d516441f0a0ef91(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:13,997 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:13,997 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416133814; duration=0sec 2024-11-24T02:42:13,998 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:13,998 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:42:14,021 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:14,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:14,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416133992, max timestamp=1732416134018Num uniques 859; 2024-11-24T02:42:14,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T02:42:14,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416133992, max timestamp=1732416134018Num uniques 859; 2024-11-24T02:42:14,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:14,026 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:14,027 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:14,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:14,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416133992, max timestamp=1732416134018Num uniques 859; 2024-11-24T02:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:42:14,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:14,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416194058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416194058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416194060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416194074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:42:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416194197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416194197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416194199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416194199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d2723a62f67643e28ce8cdfe9f1b2544 2024-11-24T02:42:14,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:42:14,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a49c146ff32d4d5e8afde41bfd9554e8 is 50, key is test_row_0/B:col10/1732416133758/Put/seqid=0 2024-11-24T02:42:14,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:14,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741867_1043 (size=12151) 2024-11-24T02:42:14,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a49c146ff32d4d5e8afde41bfd9554e8 2024-11-24T02:42:14,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f9127307e81b4ef28173476d9c2a5361 is 50, key is test_row_0/C:col10/1732416133758/Put/seqid=0 2024-11-24T02:42:14,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416194410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416194410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416194410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416194410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741868_1044 (size=12151) 2024-11-24T02:42:14,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:14,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:42:14,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416194729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416194730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416194730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416194730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:14,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:14,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f9127307e81b4ef28173476d9c2a5361 2024-11-24T02:42:14,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d2723a62f67643e28ce8cdfe9f1b2544 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544 2024-11-24T02:42:14,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T02:42:14,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a49c146ff32d4d5e8afde41bfd9554e8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8 2024-11-24T02:42:14,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T02:42:14,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f9127307e81b4ef28173476d9c2a5361 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361 2024-11-24T02:42:14,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T02:42:14,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 94fe1d236a5fa4edae1b3d10d452f90e in 981ms, sequenceid=182, compaction requested=true 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:14,878 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:14,878 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:14,880 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:14,881 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. because compaction request was cancelled 2024-11-24T02:42:14,881 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:14,882 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:14,882 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:14,882 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,882 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c5c94bc34d86417da1103d2c37014aaf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.8 K 2024-11-24T02:42:14,883 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c94bc34d86417da1103d2c37014aaf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732416132820 2024-11-24T02:42:14,884 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d46724c148ff4f2489af204f19de97d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416133458 2024-11-24T02:42:14,886 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f9127307e81b4ef28173476d9c2a5361, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732416133733 2024-11-24T02:42:14,904 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:14,905 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5c057fd09b83417a8ace093aa8755c33 is 50, key is test_row_0/C:col10/1732416133758/Put/seqid=0 2024-11-24T02:42:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741869_1045 (size=12493) 2024-11-24T02:42:14,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:42:14,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:14,979 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:14,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/431eac284d084145bd46120ff24d9f8e is 50, key is test_row_0/A:col10/1732416134071/Put/seqid=0 2024-11-24T02:42:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741870_1046 (size=12151) 2024-11-24T02:42:15,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/431eac284d084145bd46120ff24d9f8e 2024-11-24T02:42:15,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f684591f84f84385a7366baefc6ab830 is 50, key is test_row_0/B:col10/1732416134071/Put/seqid=0 2024-11-24T02:42:15,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741871_1047 (size=12151) 2024-11-24T02:42:15,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f684591f84f84385a7366baefc6ab830 2024-11-24T02:42:15,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/de40cf8994624f12add42824665b1a51 is 50, key is test_row_0/C:col10/1732416134071/Put/seqid=0 2024-11-24T02:42:15,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741872_1048 (size=12151) 2024-11-24T02:42:15,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/de40cf8994624f12add42824665b1a51 2024-11-24T02:42:15,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/431eac284d084145bd46120ff24d9f8e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e 2024-11-24T02:42:15,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e, entries=150, sequenceid=217, filesize=11.9 K 2024-11-24T02:42:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f684591f84f84385a7366baefc6ab830 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830 2024-11-24T02:42:15,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830, entries=150, sequenceid=217, filesize=11.9 K 2024-11-24T02:42:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/de40cf8994624f12add42824665b1a51 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51 2024-11-24T02:42:15,148 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51, entries=150, sequenceid=217, filesize=11.9 K 2024-11-24T02:42:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for 94fe1d236a5fa4edae1b3d10d452f90e in 171ms, sequenceid=217, compaction requested=true 2024-11-24T02:42:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T02:42:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T02:42:15,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T02:42:15,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1260 sec 2024-11-24T02:42:15,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1400 sec 2024-11-24T02:42:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:42:15,170 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:15,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T02:42:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:15,174 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:15,175 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:15,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:15,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:42:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:15,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:15,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:15,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cf21e30da4424ea8a6f221d30a9afc39 is 50, key is test_row_0/A:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:15,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741873_1049 (size=12151) 2024-11-24T02:42:15,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732416135254, max timestamp=1732416135272Num uniques 859; 2024-11-24T02:42:15,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732416135254, max timestamp=1732416135272Num uniques 859; 2024-11-24T02:42:15,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1732416135254, max timestamp=1732416135272Num uniques 859; 2024-11-24T02:42:15,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-24T02:42:15,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-24T02:42:15,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-24T02:42:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:15,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416195293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416195295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416195296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416195298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416195300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:15,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,335 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5c057fd09b83417a8ace093aa8755c33 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c057fd09b83417a8ace093aa8755c33 2024-11-24T02:42:15,346 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into 5c057fd09b83417a8ace093aa8755c33(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:15,346 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:15,347 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416134878; duration=0sec 2024-11-24T02:42:15,347 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:15,347 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416195409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416195410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416195410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416195410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416195410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:15,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416195619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416195619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416195620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416195620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416195620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:15,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:15,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cf21e30da4424ea8a6f221d30a9afc39 2024-11-24T02:42:15,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1e77df7126c5410a98cf5b6134f5748d is 50, key is test_row_0/B:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741874_1050 (size=12151) 2024-11-24T02:42:15,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:15,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:15,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416195929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416195929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416195939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416195939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416195939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:15,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:15,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:15,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1e77df7126c5410a98cf5b6134f5748d 2024-11-24T02:42:16,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/58e41adf706943a29c186c953a34ee50 is 50, key is test_row_0/C:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:16,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741875_1051 (size=12151) 2024-11-24T02:42:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:16,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:16,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416196439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:16,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416196439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:16,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416196450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:16,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416196450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:16,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:16,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416196450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:16,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:16,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/58e41adf706943a29c186c953a34ee50 2024-11-24T02:42:16,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/cf21e30da4424ea8a6f221d30a9afc39 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39 2024-11-24T02:42:16,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:42:16,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1e77df7126c5410a98cf5b6134f5748d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d 2024-11-24T02:42:16,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:42:16,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/58e41adf706943a29c186c953a34ee50 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50 2024-11-24T02:42:16,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:42:16,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 94fe1d236a5fa4edae1b3d10d452f90e in 1325ms, sequenceid=227, compaction requested=true 2024-11-24T02:42:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:16,557 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:16,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:42:16,558 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:16,560 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:16,561 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:16,561 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,561 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c039fb84848b4cbe98e64a868af82c6b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=47.8 K 2024-11-24T02:42:16,562 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:16,562 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:16,562 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,562 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/6c5287f675d940648d516441f0a0ef91, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=47.8 K 2024-11-24T02:42:16,562 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c039fb84848b4cbe98e64a868af82c6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416133458 2024-11-24T02:42:16,563 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5287f675d940648d516441f0a0ef91, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416133458 2024-11-24T02:42:16,563 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2723a62f67643e28ce8cdfe9f1b2544, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732416133733 2024-11-24T02:42:16,565 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a49c146ff32d4d5e8afde41bfd9554e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732416133733 2024-11-24T02:42:16,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:42:16,565 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 431eac284d084145bd46120ff24d9f8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732416134038 2024-11-24T02:42:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,566 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f684591f84f84385a7366baefc6ab830, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732416134038 2024-11-24T02:42:16,566 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=140.89 KB heapSize=376.78 KB 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:16,566 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf21e30da4424ea8a6f221d30a9afc39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:16,567 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e77df7126c5410a98cf5b6134f5748d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:16,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/65dae36e9af243928a9faae714c30656 is 50, key is test_row_0/A:col10/1732416135292/Put/seqid=0 2024-11-24T02:42:16,611 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:16,612 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/7777c580f3c6467d8358f9fdd14d743d is 50, key is test_row_0/A:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:16,615 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:16,616 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/8fecb3fd0da94257a3ff70880193d14c is 50, key is test_row_0/B:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:16,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741876_1052 (size=12251) 2024-11-24T02:42:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741877_1053 (size=12595) 2024-11-24T02:42:16,637 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/7777c580f3c6467d8358f9fdd14d743d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/7777c580f3c6467d8358f9fdd14d743d 2024-11-24T02:42:16,649 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 7777c580f3c6467d8358f9fdd14d743d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741878_1054 (size=12595) 2024-11-24T02:42:16,649 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:16,649 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=12, startTime=1732416136557; duration=0sec 2024-11-24T02:42:16,650 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:16,650 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:16,650 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:16,651 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:16,651 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:16,651 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:16,652 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c057fd09b83417a8ace093aa8755c33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=35.9 K 2024-11-24T02:42:16,653 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c057fd09b83417a8ace093aa8755c33, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732416133733 2024-11-24T02:42:16,654 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting de40cf8994624f12add42824665b1a51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732416134038 2024-11-24T02:42:16,658 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e41adf706943a29c186c953a34ee50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:16,658 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/8fecb3fd0da94257a3ff70880193d14c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/8fecb3fd0da94257a3ff70880193d14c 2024-11-24T02:42:16,671 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 8fecb3fd0da94257a3ff70880193d14c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:16,671 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:16,671 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=12, startTime=1732416136557; duration=0sec 2024-11-24T02:42:16,672 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:16,672 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:16,699 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:16,702 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/ed4b871f8e614b55983829068ddcf0a1 is 50, key is test_row_0/C:col10/1732416135227/Put/seqid=0 2024-11-24T02:42:16,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741879_1055 (size=12595) 2024-11-24T02:42:17,018 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/65dae36e9af243928a9faae714c30656 2024-11-24T02:42:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/50bffedeab1c4aa98d0902e190b75f27 is 50, key is test_row_0/B:col10/1732416135292/Put/seqid=0 2024-11-24T02:42:17,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741880_1056 (size=12251) 2024-11-24T02:42:17,053 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/50bffedeab1c4aa98d0902e190b75f27 2024-11-24T02:42:17,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/db55856c9e484837ae81599681107fc7 is 50, key is test_row_0/C:col10/1732416135292/Put/seqid=0 2024-11-24T02:42:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741881_1057 (size=12251) 2024-11-24T02:42:17,138 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/ed4b871f8e614b55983829068ddcf0a1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/ed4b871f8e614b55983829068ddcf0a1 2024-11-24T02:42:17,147 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into ed4b871f8e614b55983829068ddcf0a1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:17,147 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:17,147 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416136558; duration=0sec 2024-11-24T02:42:17,147 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:17,148 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:17,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416197461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416197466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416197466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416197467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416197468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,473 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in 94fe1d236a5fa4edae1b3d10d452f90e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/db55856c9e484837ae81599681107fc7 2024-11-24T02:42:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/65dae36e9af243928a9faae714c30656 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656 2024-11-24T02:42:17,519 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656, entries=150, sequenceid=260, filesize=12.0 K 2024-11-24T02:42:17,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/50bffedeab1c4aa98d0902e190b75f27 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27 2024-11-24T02:42:17,531 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27, entries=150, sequenceid=260, filesize=12.0 K 2024-11-24T02:42:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/db55856c9e484837ae81599681107fc7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7 2024-11-24T02:42:17,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7, entries=150, sequenceid=260, filesize=12.0 K 2024-11-24T02:42:17,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~376.73 KB/385776, currentSize=46.96 KB/48090 for 94fe1d236a5fa4edae1b3d10d452f90e in 976ms, sequenceid=260, compaction requested=false 2024-11-24T02:42:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:17,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T02:42:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T02:42:17,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T02:42:17,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-11-24T02:42:17,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3790 sec 2024-11-24T02:42:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:17,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:17,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/73471c547e6141328610105b06f2d00d is 50, key is test_row_0/A:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:17,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741882_1058 (size=14741) 2024-11-24T02:42:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416197608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416197616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416137596, max timestamp=1732416137619Num uniques 859; 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416137596, max timestamp=1732416137619Num uniques 859; 2024-11-24T02:42:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416197619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416137596, max timestamp=1732416137619Num uniques 859; 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:42:17,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416197689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416197729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416197729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416197729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416197799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416197940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416197939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416197940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:17,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/73471c547e6141328610105b06f2d00d 2024-11-24T02:42:18,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/78b42699eefc4dbf881c39f986958229 is 50, key is test_row_0/B:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416198009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741883_1059 (size=12301) 2024-11-24T02:42:18,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/78b42699eefc4dbf881c39f986958229 2024-11-24T02:42:18,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5c080e68bd9a4ed2b78eeabb696311fb is 50, key is test_row_0/C:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:18,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741884_1060 (size=12301) 2024-11-24T02:42:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416198249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416198249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416198249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416198319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5c080e68bd9a4ed2b78eeabb696311fb 2024-11-24T02:42:18,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/73471c547e6141328610105b06f2d00d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d 2024-11-24T02:42:18,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d, entries=200, sequenceid=274, filesize=14.4 K 2024-11-24T02:42:18,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/78b42699eefc4dbf881c39f986958229 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229 2024-11-24T02:42:18,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:42:18,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5c080e68bd9a4ed2b78eeabb696311fb as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb 2024-11-24T02:42:18,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:42:18,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 94fe1d236a5fa4edae1b3d10d452f90e in 916ms, sequenceid=274, compaction requested=true 2024-11-24T02:42:18,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:18,489 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:18,489 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:18,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:18,490 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:18,490 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39587 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:18,490 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:18,490 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:18,490 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:18,490 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:18,491 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/8fecb3fd0da94257a3ff70880193d14c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=36.3 K 2024-11-24T02:42:18,491 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/7777c580f3c6467d8358f9fdd14d743d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=38.7 K 2024-11-24T02:42:18,491 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fecb3fd0da94257a3ff70880193d14c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:18,492 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7777c580f3c6467d8358f9fdd14d743d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:18,492 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 50bffedeab1c4aa98d0902e190b75f27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416135292 2024-11-24T02:42:18,495 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 78b42699eefc4dbf881c39f986958229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:18,495 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65dae36e9af243928a9faae714c30656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416135292 2024-11-24T02:42:18,496 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73471c547e6141328610105b06f2d00d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:18,515 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:18,516 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9f5446bf2d5341f6a14dd53be7fc4165 is 50, key is test_row_0/B:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:18,522 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:18,523 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ad48256046c847a49b4e551b71468a82 is 50, key is test_row_0/A:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:18,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741885_1061 (size=12847) 2024-11-24T02:42:18,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741886_1062 (size=12847) 2024-11-24T02:42:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:18,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:18,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:18,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/9e4a5cd0d20642829302696ed7ec1b27 is 50, key is test_row_0/A:col10/1732416137687/Put/seqid=0 2024-11-24T02:42:18,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741887_1063 (size=12301) 2024-11-24T02:42:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416198781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416198782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416198788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416198829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416198899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416198899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416198899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:18,953 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9f5446bf2d5341f6a14dd53be7fc4165 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f5446bf2d5341f6a14dd53be7fc4165 2024-11-24T02:42:18,955 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ad48256046c847a49b4e551b71468a82 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ad48256046c847a49b4e551b71468a82 2024-11-24T02:42:18,960 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 9f5446bf2d5341f6a14dd53be7fc4165(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:18,960 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:18,961 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416138489; duration=0sec 2024-11-24T02:42:18,961 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:18,961 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:18,961 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:18,962 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into ad48256046c847a49b4e551b71468a82(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:18,962 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:18,962 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416138489; duration=0sec 2024-11-24T02:42:18,962 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:18,962 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:18,963 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:18,963 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:18,963 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:18,963 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/ed4b871f8e614b55983829068ddcf0a1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=36.3 K 2024-11-24T02:42:18,963 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4b871f8e614b55983829068ddcf0a1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416135209 2024-11-24T02:42:18,964 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting db55856c9e484837ae81599681107fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416135292 2024-11-24T02:42:18,964 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c080e68bd9a4ed2b78eeabb696311fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:18,972 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:18,972 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b8cdb5139a824f3290b992502a2cd689 is 50, key is test_row_0/C:col10/1732416137464/Put/seqid=0 2024-11-24T02:42:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741888_1064 (size=12847) 2024-11-24T02:42:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416199109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416199109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416199109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/9e4a5cd0d20642829302696ed7ec1b27 2024-11-24T02:42:19,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9f6920300bcd4000ac6bfe42ab7b290d is 50, key is test_row_0/B:col10/1732416137687/Put/seqid=0 2024-11-24T02:42:19,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741889_1065 (size=12301) 2024-11-24T02:42:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:42:19,329 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:19,330 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T02:42:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:19,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:19,332 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:19,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:19,393 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b8cdb5139a824f3290b992502a2cd689 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b8cdb5139a824f3290b992502a2cd689 2024-11-24T02:42:19,406 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into b8cdb5139a824f3290b992502a2cd689(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:19,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:19,406 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416138489; duration=0sec 2024-11-24T02:42:19,407 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:19,407 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416199419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416199419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416199420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:19,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:42:19,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:19,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416199499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,501 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in 94fe1d236a5fa4edae1b3d10d452f90e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:19,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9f6920300bcd4000ac6bfe42ab7b290d 2024-11-24T02:42:19,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5ac1a856be8d48b9afb21804d01b02e6 is 50, key is test_row_0/C:col10/1732416137687/Put/seqid=0 2024-11-24T02:42:19,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741890_1066 (size=12301) 2024-11-24T02:42:19,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:42:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:19,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:42:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416199839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,840 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in 94fe1d236a5fa4edae1b3d10d452f90e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416199929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416199929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416199929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:19,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:42:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:19,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:20,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5ac1a856be8d48b9afb21804d01b02e6 2024-11-24T02:42:20,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/9e4a5cd0d20642829302696ed7ec1b27 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27 2024-11-24T02:42:20,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27, entries=150, sequenceid=309, filesize=12.0 K 2024-11-24T02:42:20,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/9f6920300bcd4000ac6bfe42ab7b290d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d 2024-11-24T02:42:20,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d, entries=150, sequenceid=309, filesize=12.0 K 2024-11-24T02:42:20,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5ac1a856be8d48b9afb21804d01b02e6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6 2024-11-24T02:42:20,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6, entries=150, sequenceid=309, filesize=12.0 K 2024-11-24T02:42:20,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 94fe1d236a5fa4edae1b3d10d452f90e in 1291ms, sequenceid=309, compaction requested=false 2024-11-24T02:42:20,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:20,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:20,102 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/2cc5a69c2aa64d5fa90da4b621f82d46 is 50, key is test_row_0/A:col10/1732416138786/Put/seqid=0 2024-11-24T02:42:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741891_1067 (size=12301) 2024-11-24T02:42:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:20,513 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/2cc5a69c2aa64d5fa90da4b621f82d46 2024-11-24T02:42:20,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/70fb6770b6f04409939b680a16240a96 is 50, key is test_row_0/B:col10/1732416138786/Put/seqid=0 2024-11-24T02:42:20,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741892_1068 (size=12301) 2024-11-24T02:42:20,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/70fb6770b6f04409939b680a16240a96 2024-11-24T02:42:20,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 is 50, key is test_row_0/C:col10/1732416138786/Put/seqid=0 2024-11-24T02:42:20,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741893_1069 (size=12301) 2024-11-24T02:42:20,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:20,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416140960, max timestamp=1732416140978Num uniques 859; 2024-11-24T02:42:20,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416140960, max timestamp=1732416140978Num uniques 859; 2024-11-24T02:42:20,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:42:20,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:42:20,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416140960, max timestamp=1732416140978Num uniques 859; 2024-11-24T02:42:20,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:21,008 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 2024-11-24T02:42:21,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/2cc5a69c2aa64d5fa90da4b621f82d46 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46 2024-11-24T02:42:21,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416201024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416201029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416201030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46, entries=150, sequenceid=322, filesize=12.0 K 2024-11-24T02:42:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/70fb6770b6f04409939b680a16240a96 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96 2024-11-24T02:42:21,041 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96, entries=150, sequenceid=322, filesize=12.0 K 2024-11-24T02:42:21,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 2024-11-24T02:42:21,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806, entries=150, sequenceid=322, filesize=12.0 K 2024-11-24T02:42:21,048 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 94fe1d236a5fa4edae1b3d10d452f90e in 946ms, sequenceid=322, compaction requested=true 2024-11-24T02:42:21,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T02:42:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T02:42:21,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T02:42:21,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-24T02:42:21,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7230 sec 2024-11-24T02:42:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:21,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416140999, max timestamp=1732416141144Num uniques 1018; 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:42:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416140999, max timestamp=1732416141144Num uniques 1168; 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416140999, max timestamp=1732416141144Num uniques 1168; 2024-11-24T02:42:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-24T02:42:21,149 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-24T02:42:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:21,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/175305a1d986490fb93a9d4d8ed3231b is 50, key is test_row_0/A:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741894_1070 (size=12301) 2024-11-24T02:42:21,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/175305a1d986490fb93a9d4d8ed3231b 2024-11-24T02:42:21,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/140f98b990e94acbbacf7cf5d377dbd0 is 50, key is test_row_0/B:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416141180, max timestamp=1732416141190Num uniques 859; 2024-11-24T02:42:21,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:21,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416141180, max timestamp=1732416141190Num uniques 859; 2024-11-24T02:42:21,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:21,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416141180, max timestamp=1732416141190Num uniques 859; 2024-11-24T02:42:21,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:21,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741895_1071 (size=12301) 2024-11-24T02:42:21,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/140f98b990e94acbbacf7cf5d377dbd0 2024-11-24T02:42:21,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d2a56303b054c40a54bab8364740c18 is 50, key is test_row_0/C:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741896_1072 (size=12301) 2024-11-24T02:42:21,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d2a56303b054c40a54bab8364740c18 2024-11-24T02:42:21,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/175305a1d986490fb93a9d4d8ed3231b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b 2024-11-24T02:42:21,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b, entries=150, sequenceid=360, filesize=12.0 K 2024-11-24T02:42:21,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/140f98b990e94acbbacf7cf5d377dbd0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0 2024-11-24T02:42:21,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732416141231, max timestamp=1732416141255Num uniques 1018; 2024-11-24T02:42:21,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=16848, suffix heap size=133840,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=368 2024-11-24T02:42:21,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416141236, max timestamp=1732416141255Num uniques 1018; 2024-11-24T02:42:21,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416141236, max timestamp=1732416141255Num uniques 1018; 2024-11-24T02:42:21,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:42:21,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:42:21,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0, entries=150, sequenceid=360, filesize=12.0 K 2024-11-24T02:42:21,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d2a56303b054c40a54bab8364740c18 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18 2024-11-24T02:42:21,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18, entries=150, sequenceid=360, filesize=12.0 K 2024-11-24T02:42:21,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=40.25 KB/41220 for 94fe1d236a5fa4edae1b3d10d452f90e in 155ms, sequenceid=360, compaction requested=true 2024-11-24T02:42:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,301 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:21,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:21,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:21,303 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:21,303 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:21,303 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:21,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=60.38 KB heapSize=155.63 KB 2024-11-24T02:42:21,303 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,303 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ad48256046c847a49b4e551b71468a82, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=48.6 K 2024-11-24T02:42:21,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:21,304 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad48256046c847a49b4e551b71468a82, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:21,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:21,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,305 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4a5cd0d20642829302696ed7ec1b27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732416137670 2024-11-24T02:42:21,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:21,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,305 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:21,305 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:21,305 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,305 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f5446bf2d5341f6a14dd53be7fc4165, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=48.6 K 2024-11-24T02:42:21,306 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc5a69c2aa64d5fa90da4b621f82d46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416138776 2024-11-24T02:42:21,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5446bf2d5341f6a14dd53be7fc4165, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:21,306 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 175305a1d986490fb93a9d4d8ed3231b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:21,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f6920300bcd4000ac6bfe42ab7b290d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732416137670 2024-11-24T02:42:21,307 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 70fb6770b6f04409939b680a16240a96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416138776 2024-11-24T02:42:21,308 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 140f98b990e94acbbacf7cf5d377dbd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:21,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/3740bc93d5b14f79880e95d04b239009 is 50, key is test_row_0/A:col10/1732416141282/Put/seqid=0 2024-11-24T02:42:21,331 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:21,332 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/e8774e943dc24e259de55eb2ecae5810 is 50, key is test_row_0/B:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,336 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:21,337 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c558257adea542cd90fb829e657dd8dd is 50, key is test_row_0/A:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741897_1073 (size=12301) 2024-11-24T02:42:21,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/3740bc93d5b14f79880e95d04b239009 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416141341, max timestamp=1732416141370Num uniques 859; 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416141341, max timestamp=1732416141370Num uniques 859; 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416141341, max timestamp=1732416141370Num uniques 859; 2024-11-24T02:42:21,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:21,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/c6e3ac3231fc4210b04c59ba164b0275 is 50, key is test_row_0/B:col10/1732416141282/Put/seqid=0 2024-11-24T02:42:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741898_1074 (size=12983) 2024-11-24T02:42:21,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741899_1075 (size=12983) 2024-11-24T02:42:21,407 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c558257adea542cd90fb829e657dd8dd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c558257adea542cd90fb829e657dd8dd 2024-11-24T02:42:21,424 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into c558257adea542cd90fb829e657dd8dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:21,424 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,425 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=12, startTime=1732416141301; duration=0sec 2024-11-24T02:42:21,425 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:21,425 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:21,425 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:21,427 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:21,427 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:21,427 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,427 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b8cdb5139a824f3290b992502a2cd689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=48.6 K 2024-11-24T02:42:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,428 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8cdb5139a824f3290b992502a2cd689, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416137463 2024-11-24T02:42:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416201426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416201427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,430 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac1a856be8d48b9afb21804d01b02e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732416137670 2024-11-24T02:42:21,430 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bcbb3ee2c84fb7b48e1f3ec6a8d806, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416138776 2024-11-24T02:42:21,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741900_1076 (size=12301) 2024-11-24T02:42:21,431 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2a56303b054c40a54bab8364740c18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:21,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/c6e3ac3231fc4210b04c59ba164b0275 2024-11-24T02:42:21,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d5037486c15a470d92ed7519c33f1145 is 50, key is test_row_0/C:col10/1732416141282/Put/seqid=0 2024-11-24T02:42:21,442 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:21,443 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/a5fef007890a4beb8b24c3963d08d5d8 is 50, key is test_row_0/C:col10/1732416141027/Put/seqid=0 2024-11-24T02:42:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416201429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741901_1077 (size=12301) 2024-11-24T02:42:21,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d5037486c15a470d92ed7519c33f1145 2024-11-24T02:42:21,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/3740bc93d5b14f79880e95d04b239009 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009 2024-11-24T02:42:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:42:21,480 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:21,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:21,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009, entries=150, sequenceid=401, filesize=12.0 K 2024-11-24T02:42:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T02:42:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:21,483 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:21,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741902_1078 (size=12983) 2024-11-24T02:42:21,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/c6e3ac3231fc4210b04c59ba164b0275 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275 2024-11-24T02:42:21,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:21,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:21,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275, entries=150, sequenceid=401, filesize=12.0 K 2024-11-24T02:42:21,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d5037486c15a470d92ed7519c33f1145 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145 2024-11-24T02:42:21,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145, entries=150, sequenceid=401, filesize=12.0 K 2024-11-24T02:42:21,505 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/a5fef007890a4beb8b24c3963d08d5d8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a5fef007890a4beb8b24c3963d08d5d8 2024-11-24T02:42:21,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~155.58 KB/159312, currentSize=134.18 KB/137400 for 94fe1d236a5fa4edae1b3d10d452f90e in 203ms, sequenceid=401, compaction requested=false 2024-11-24T02:42:21,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,514 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into a5fef007890a4beb8b24c3963d08d5d8(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:21,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,514 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=12, startTime=1732416141303; duration=0sec 2024-11-24T02:42:21,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:21,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:21,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:21,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/471c1b8537614c3aa77b0ff5f9c628b1 is 50, key is test_row_0/A:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:21,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741903_1079 (size=14741) 2024-11-24T02:42:21,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/471c1b8537614c3aa77b0ff5f9c628b1 2024-11-24T02:42:21,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416201561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f8e2a116d81543938a70b7cc08c7b2a7 is 50, key is test_row_0/B:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:21,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416201569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416201574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:21,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741904_1080 (size=12301) 2024-11-24T02:42:21,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f8e2a116d81543938a70b7cc08c7b2a7 2024-11-24T02:42:21,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/481aec059bda40eaac9f9696e366a766 is 50, key is test_row_0/C:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:21,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741905_1081 (size=12301) 2024-11-24T02:42:21,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:42:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416201679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416201689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416201699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:21,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:42:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,816 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/e8774e943dc24e259de55eb2ecae5810 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e8774e943dc24e259de55eb2ecae5810 2024-11-24T02:42:21,821 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into e8774e943dc24e259de55eb2ecae5810(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:21,821 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:21,822 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=12, startTime=1732416141302; duration=0sec 2024-11-24T02:42:21,822 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:21,822 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:21,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416201869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,871 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in 94fe1d236a5fa4edae1b3d10d452f90e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416201889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416201899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416201909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:21,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:42:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:21,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:22,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/481aec059bda40eaac9f9696e366a766 2024-11-24T02:42:22,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/471c1b8537614c3aa77b0ff5f9c628b1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1 2024-11-24T02:42:22,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1, entries=200, sequenceid=439, filesize=14.4 K 2024-11-24T02:42:22,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/f8e2a116d81543938a70b7cc08c7b2a7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7 2024-11-24T02:42:22,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7, entries=150, sequenceid=439, filesize=12.0 K 2024-11-24T02:42:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/481aec059bda40eaac9f9696e366a766 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766 2024-11-24T02:42:22,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766, entries=150, sequenceid=439, filesize=12.0 K 2024-11-24T02:42:22,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 94fe1d236a5fa4edae1b3d10d452f90e in 529ms, sequenceid=439, compaction requested=true 2024-11-24T02:42:22,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:22,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:22,075 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:22,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:42:22,076 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:22,076 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:22,076 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:22,076 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c558257adea542cd90fb829e657dd8dd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=39.1 K 2024-11-24T02:42:22,077 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:22,077 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c558257adea542cd90fb829e657dd8dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:22,078 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3740bc93d5b14f79880e95d04b239009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732416141256 2024-11-24T02:42:22,083 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:22,083 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:22,083 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:22,083 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e8774e943dc24e259de55eb2ecae5810, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=36.7 K 2024-11-24T02:42:22,084 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471c1b8537614c3aa77b0ff5f9c628b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:22,089 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e8774e943dc24e259de55eb2ecae5810, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:22,093 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e3ac3231fc4210b04c59ba164b0275, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732416141256 2024-11-24T02:42:22,096 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e2a116d81543938a70b7cc08c7b2a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:22,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:42:22,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:22,113 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:22,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:22,115 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:22,116 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/0fef887887c045999109d2a063bb06ca is 50, key is test_row_0/A:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:22,122 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:22,123 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4b074f64bdcc4dce9b933fbe7a4bb2fc is 50, key is test_row_0/B:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:22,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6b349979e12546c482c27c753bfa2929 is 50, key is test_row_0/A:col10/1732416141566/Put/seqid=0 2024-11-24T02:42:22,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741906_1082 (size=13085) 2024-11-24T02:42:22,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741907_1083 (size=13085) 2024-11-24T02:42:22,205 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/0fef887887c045999109d2a063bb06ca as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/0fef887887c045999109d2a063bb06ca 2024-11-24T02:42:22,213 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 0fef887887c045999109d2a063bb06ca(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:22,213 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:22,213 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416142074; duration=0sec 2024-11-24T02:42:22,215 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:22,215 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:22,215 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:22,219 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:22,219 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:22,219 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:22,219 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a5fef007890a4beb8b24c3963d08d5d8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=36.7 K 2024-11-24T02:42:22,220 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5fef007890a4beb8b24c3963d08d5d8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416141027 2024-11-24T02:42:22,221 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5037486c15a470d92ed7519c33f1145, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1732416141256 2024-11-24T02:42:22,221 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 481aec059bda40eaac9f9696e366a766, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:22,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741908_1084 (size=9857) 2024-11-24T02:42:22,250 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:22,251 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/aac4ea354b064670bbfba50fb3729955 is 50, key is test_row_0/C:col10/1732416141426/Put/seqid=0 2024-11-24T02:42:22,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741909_1085 (size=13085) 2024-11-24T02:42:22,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416142231, max timestamp=1732416142268Num uniques 859; 2024-11-24T02:42:22,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:22,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416142231, max timestamp=1732416142270Num uniques 859; 2024-11-24T02:42:22,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:22,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416142231, max timestamp=1732416142268Num uniques 859; 2024-11-24T02:42:22,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:22,278 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/aac4ea354b064670bbfba50fb3729955 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/aac4ea354b064670bbfba50fb3729955 2024-11-24T02:42:22,287 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into aac4ea354b064670bbfba50fb3729955(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:22,287 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:22,287 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416142075; duration=0sec 2024-11-24T02:42:22,287 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:22,287 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416202317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416202319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416202320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416202439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416202439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416202439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:22,626 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4b074f64bdcc4dce9b933fbe7a4bb2fc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4b074f64bdcc4dce9b933fbe7a4bb2fc 2024-11-24T02:42:22,627 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6b349979e12546c482c27c753bfa2929 2024-11-24T02:42:22,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/b3f51d76a7a348de968460d974946a23 is 50, key is test_row_0/B:col10/1732416141566/Put/seqid=0 2024-11-24T02:42:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416202651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,665 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 4b074f64bdcc4dce9b933fbe7a4bb2fc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:22,665 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:22,665 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416142075; duration=0sec 2024-11-24T02:42:22,666 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:22,666 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:22,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416202659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416202664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741910_1086 (size=9857) 2024-11-24T02:42:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416202969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416202979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416202979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:22,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/b3f51d76a7a348de968460d974946a23 2024-11-24T02:42:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/01585a4448664295b4dcf9591ed974d7 is 50, key is test_row_0/C:col10/1732416141566/Put/seqid=0 2024-11-24T02:42:23,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741911_1087 (size=9857) 2024-11-24T02:42:23,156 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/01585a4448664295b4dcf9591ed974d7 2024-11-24T02:42:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/6b349979e12546c482c27c753bfa2929 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929 2024-11-24T02:42:23,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929, entries=100, sequenceid=450, filesize=9.6 K 2024-11-24T02:42:23,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/b3f51d76a7a348de968460d974946a23 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23 2024-11-24T02:42:23,207 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23, entries=100, sequenceid=450, filesize=9.6 K 2024-11-24T02:42:23,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/01585a4448664295b4dcf9591ed974d7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7 2024-11-24T02:42:23,216 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7, entries=100, sequenceid=450, filesize=9.6 K 2024-11-24T02:42:23,217 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 94fe1d236a5fa4edae1b3d10d452f90e in 1104ms, sequenceid=450, compaction requested=false 2024-11-24T02:42:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:23,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T02:42:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T02:42:23,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T02:42:23,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7280 sec 2024-11-24T02:42:23,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7420 sec 2024-11-24T02:42:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732416142314, max timestamp=1732416143491Num uniques 1018; 2024-11-24T02:42:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732416142314, max timestamp=1732416143491Num uniques 1018; 2024-11-24T02:42:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T02:42:23,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T02:42:23,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:23,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:42:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:23,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ef4ea1d740ae4771aff539c2955a7c4f is 50, key is test_row_0/A:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:23,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741912_1088 (size=12301) 2024-11-24T02:42:23,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ef4ea1d740ae4771aff539c2955a7c4f 2024-11-24T02:42:23,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/58f7bd22f76d434382c938113c399b92 is 50, key is test_row_0/B:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:23,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416203596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416203598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416203600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416203608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741913_1089 (size=12301) 2024-11-24T02:42:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:42:23,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/58f7bd22f76d434382c938113c399b92 2024-11-24T02:42:23,639 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:23,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T02:42:23,643 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:23,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:42:23,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:23,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:23,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/171efe908d5a4107be7115c6db4f2c14 is 50, key is test_row_0/C:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741914_1090 (size=12301) 2024-11-24T02:42:23,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416203719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416203719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416203719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416203720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:42:23,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:23,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416203931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416203931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416203931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:23,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416203931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:23,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:23,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:23,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:23,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:42:24,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/171efe908d5a4107be7115c6db4f2c14 2024-11-24T02:42:24,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ef4ea1d740ae4771aff539c2955a7c4f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f 2024-11-24T02:42:24,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:42:24,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/58f7bd22f76d434382c938113c399b92 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92 2024-11-24T02:42:24,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:42:24,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/171efe908d5a4107be7115c6db4f2c14 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14 2024-11-24T02:42:24,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:42:24,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~178.75 KB/183040, currentSize=114.05 KB/116790 for 94fe1d236a5fa4edae1b3d10d452f90e in 657ms, sequenceid=491, compaction requested=true 2024-11-24T02:42:24,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:24,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:24,158 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:24,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:24,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:24,159 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:24,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:24,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:24,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:24,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:24,161 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,161 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/0fef887887c045999109d2a063bb06ca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=34.4 K 2024-11-24T02:42:24,161 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fef887887c045999109d2a063bb06ca, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:24,161 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:24,162 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:24,162 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,162 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4b074f64bdcc4dce9b933fbe7a4bb2fc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=34.4 K 2024-11-24T02:42:24,162 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b349979e12546c482c27c753bfa2929, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732416141566 2024-11-24T02:42:24,162 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b074f64bdcc4dce9b933fbe7a4bb2fc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:24,163 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef4ea1d740ae4771aff539c2955a7c4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:24,163 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f51d76a7a348de968460d974946a23, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732416141566 2024-11-24T02:42:24,163 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f7bd22f76d434382c938113c399b92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:24,173 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:24,174 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:24,174 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/072cb262b45441a4ac1a341a4cae94c2 is 50, key is test_row_0/B:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:24,175 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/81889dad0fa741238b0fe3e45db484b9 is 50, key is test_row_0/A:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:24,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741915_1091 (size=13187) 2024-11-24T02:42:24,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741916_1092 (size=13187) 2024-11-24T02:42:24,192 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/072cb262b45441a4ac1a341a4cae94c2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/072cb262b45441a4ac1a341a4cae94c2 2024-11-24T02:42:24,200 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 072cb262b45441a4ac1a341a4cae94c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:24,200 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,200 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416144158; duration=0sec 2024-11-24T02:42:24,203 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:24,203 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:24,203 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:24,206 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:24,206 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:24,206 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,206 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/aac4ea354b064670bbfba50fb3729955, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=34.4 K 2024-11-24T02:42:24,209 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting aac4ea354b064670bbfba50fb3729955, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732416141414 2024-11-24T02:42:24,210 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 01585a4448664295b4dcf9591ed974d7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732416141566 2024-11-24T02:42:24,210 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 171efe908d5a4107be7115c6db4f2c14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:24,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:24,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,244 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#80 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:24,247 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/c7ad6cdb8d9643bc84e8c0e86410aaa7 is 50, key is test_row_0/C:col10/1732416142314/Put/seqid=0 2024-11-24T02:42:24,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fd9ea872868d4298927336d18e6fe481 is 50, key is test_row_0/A:col10/1732416143594/Put/seqid=0 2024-11-24T02:42:24,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:24,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:42:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416204266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416204268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741917_1093 (size=13187) 2024-11-24T02:42:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416204277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416204277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,294 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/c7ad6cdb8d9643bc84e8c0e86410aaa7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c7ad6cdb8d9643bc84e8c0e86410aaa7 2024-11-24T02:42:24,310 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into c7ad6cdb8d9643bc84e8c0e86410aaa7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:24,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,310 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416144159; duration=0sec 2024-11-24T02:42:24,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:24,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:24,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741918_1094 (size=14741) 2024-11-24T02:42:24,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fd9ea872868d4298927336d18e6fe481 2024-11-24T02:42:24,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a988d5a26d7c4d8bbce8ade021126b86 is 50, key is test_row_0/B:col10/1732416143594/Put/seqid=0 2024-11-24T02:42:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416204389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416204389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416204389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416204389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741919_1095 (size=12301) 2024-11-24T02:42:24,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a988d5a26d7c4d8bbce8ade021126b86 2024-11-24T02:42:24,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b22f44cb33b74e2ba67710efe458caf8 is 50, key is test_row_0/C:col10/1732416143594/Put/seqid=0 2024-11-24T02:42:24,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741920_1096 (size=12301) 2024-11-24T02:42:24,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b22f44cb33b74e2ba67710efe458caf8 2024-11-24T02:42:24,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fd9ea872868d4298927336d18e6fe481 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481 2024-11-24T02:42:24,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481, entries=200, sequenceid=513, filesize=14.4 K 2024-11-24T02:42:24,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/a988d5a26d7c4d8bbce8ade021126b86 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86 2024-11-24T02:42:24,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86, entries=150, sequenceid=513, filesize=12.0 K 2024-11-24T02:42:24,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/b22f44cb33b74e2ba67710efe458caf8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8 2024-11-24T02:42:24,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8, entries=150, sequenceid=513, filesize=12.0 K 2024-11-24T02:42:24,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 94fe1d236a5fa4edae1b3d10d452f90e in 266ms, sequenceid=513, compaction requested=false 2024-11-24T02:42:24,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:42:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,578 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:24,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:24,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c0048ec393d74e509ae9778193e02e5d is 50, key is test_row_0/A:col10/1732416144264/Put/seqid=0 2024-11-24T02:42:24,594 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/81889dad0fa741238b0fe3e45db484b9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/81889dad0fa741238b0fe3e45db484b9 2024-11-24T02:42:24,605 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 81889dad0fa741238b0fe3e45db484b9(size=12.9 K), total size for store is 27.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:24,606 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,606 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416144158; duration=0sec 2024-11-24T02:42:24,606 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:24,606 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:24,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:24,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741921_1097 (size=12301) 2024-11-24T02:42:24,617 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c0048ec393d74e509ae9778193e02e5d 2024-11-24T02:42:24,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/41678e591a744345b38f784ea7ed6171 is 50, key is test_row_0/B:col10/1732416144264/Put/seqid=0 2024-11-24T02:42:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741922_1098 (size=12301) 2024-11-24T02:42:24,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/41678e591a744345b38f784ea7ed6171 2024-11-24T02:42:24,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416204647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416204649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416204650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416204661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/6824c0bffa624e938a2fce819a69a257 is 50, key is test_row_0/C:col10/1732416144264/Put/seqid=0 2024-11-24T02:42:24,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741923_1099 (size=12301) 2024-11-24T02:42:24,699 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/6824c0bffa624e938a2fce819a69a257 2024-11-24T02:42:24,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/c0048ec393d74e509ae9778193e02e5d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d 2024-11-24T02:42:24,721 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T02:42:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/41678e591a744345b38f784ea7ed6171 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171 2024-11-24T02:42:24,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T02:42:24,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/6824c0bffa624e938a2fce819a69a257 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257 2024-11-24T02:42:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257, entries=150, sequenceid=526, filesize=12.0 K 2024-11-24T02:42:24,750 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 94fe1d236a5fa4edae1b3d10d452f90e in 172ms, sequenceid=526, compaction requested=true 2024-11-24T02:42:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T02:42:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T02:42:24,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T02:42:24,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-24T02:42:24,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1140 sec 2024-11-24T02:42:24,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T02:42:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:24,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:24,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:24,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/17946d34a4ff43e6b211543b15a4654d is 50, key is test_row_0/A:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:42:24,779 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:24,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T02:42:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:24,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:24,783 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:24,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:24,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416204785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416204785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416204786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416204791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741924_1100 (size=14741) 2024-11-24T02:42:24,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/17946d34a4ff43e6b211543b15a4654d 2024-11-24T02:42:24,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/5a437567817e49809a72018b494ab596 is 50, key is test_row_0/B:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:24,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741925_1101 (size=12301) 2024-11-24T02:42:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:24,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416204899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416204899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416204899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416204910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:24,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:24,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:24,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:25,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416205109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416205109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416205110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416205119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/5a437567817e49809a72018b494ab596 2024-11-24T02:42:25,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/778cb49cf20a4371a0c0b51c8292907d is 50, key is test_row_0/C:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:25,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741926_1102 (size=12301) 2024-11-24T02:42:25,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416205419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416205420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416205420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416205429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/778cb49cf20a4371a0c0b51c8292907d 2024-11-24T02:42:25,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/17946d34a4ff43e6b211543b15a4654d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d 2024-11-24T02:42:25,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d, entries=200, sequenceid=548, filesize=14.4 K 2024-11-24T02:42:25,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/5a437567817e49809a72018b494ab596 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596 2024-11-24T02:42:25,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596, entries=150, sequenceid=548, filesize=12.0 K 2024-11-24T02:42:25,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/778cb49cf20a4371a0c0b51c8292907d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d 2024-11-24T02:42:25,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:25,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d, entries=150, sequenceid=548, filesize=12.0 K 2024-11-24T02:42:25,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 94fe1d236a5fa4edae1b3d10d452f90e in 946ms, sequenceid=548, compaction requested=true 2024-11-24T02:42:25,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:25,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:25,718 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:25,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:25,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:25,718 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:25,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:25,720 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:25,720 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:25,720 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,720 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/81889dad0fa741238b0fe3e45db484b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=53.7 K 2024-11-24T02:42:25,721 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81889dad0fa741238b0fe3e45db484b9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:25,721 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:25,721 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:25,722 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,722 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/072cb262b45441a4ac1a341a4cae94c2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=48.9 K 2024-11-24T02:42:25,722 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9ea872868d4298927336d18e6fe481, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732416143594 2024-11-24T02:42:25,724 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 072cb262b45441a4ac1a341a4cae94c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:25,725 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0048ec393d74e509ae9778193e02e5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732416144251 2024-11-24T02:42:25,726 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a988d5a26d7c4d8bbce8ade021126b86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732416143594 2024-11-24T02:42:25,726 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17946d34a4ff43e6b211543b15a4654d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:25,727 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 41678e591a744345b38f784ea7ed6171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732416144251 2024-11-24T02:42:25,727 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a437567817e49809a72018b494ab596, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:25,745 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:25,746 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d78cd9ae21134655818c3963272cfaf4 is 50, key is test_row_0/A:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:25,755 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:25,755 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/fa15dc83537448e48cbb86de0fdcff43 is 50, key is test_row_0/B:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:25,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741927_1103 (size=13323) 2024-11-24T02:42:25,802 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/d78cd9ae21134655818c3963272cfaf4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d78cd9ae21134655818c3963272cfaf4 2024-11-24T02:42:25,815 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into d78cd9ae21134655818c3963272cfaf4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:25,815 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:25,815 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=12, startTime=1732416145718; duration=0sec 2024-11-24T02:42:25,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:25,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:25,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741928_1104 (size=13323) 2024-11-24T02:42:25,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:25,818 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:25,819 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,819 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c7ad6cdb8d9643bc84e8c0e86410aaa7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=48.9 K 2024-11-24T02:42:25,820 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7ad6cdb8d9643bc84e8c0e86410aaa7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416142314 2024-11-24T02:42:25,821 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b22f44cb33b74e2ba67710efe458caf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1732416143594 2024-11-24T02:42:25,821 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6824c0bffa624e938a2fce819a69a257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1732416144251 2024-11-24T02:42:25,821 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 778cb49cf20a4371a0c0b51c8292907d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:25,832 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/fa15dc83537448e48cbb86de0fdcff43 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/fa15dc83537448e48cbb86de0fdcff43 2024-11-24T02:42:25,846 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into fa15dc83537448e48cbb86de0fdcff43(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:25,846 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:25,846 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=12, startTime=1732416145718; duration=0sec 2024-11-24T02:42:25,846 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:25,846 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:25,851 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:25,852 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/8025223492f74914b050986f7f0fc6bc is 50, key is test_row_0/C:col10/1732416144648/Put/seqid=0 2024-11-24T02:42:25,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:42:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:25,862 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:42:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:25,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:25,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:25,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:25,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:25,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741929_1105 (size=13323) 2024-11-24T02:42:25,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e is 50, key is test_row_0/A:col10/1732416144783/Put/seqid=0 2024-11-24T02:42:25,905 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/8025223492f74914b050986f7f0fc6bc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8025223492f74914b050986f7f0fc6bc 2024-11-24T02:42:25,917 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into 8025223492f74914b050986f7f0fc6bc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:25,917 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:25,917 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=12, startTime=1732416145718; duration=0sec 2024-11-24T02:42:25,917 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:25,917 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:25,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741930_1106 (size=12301) 2024-11-24T02:42:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416205954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416145944, max timestamp=1732416145960Num uniques 859; 2024-11-24T02:42:25,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416145944, max timestamp=1732416145960Num uniques 859; 2024-11-24T02:42:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:25,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416205982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416205984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416205986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416205987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416206068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416206099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416206100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416206100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416206100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416206279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416206309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416206309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416206310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416206310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e 2024-11-24T02:42:26,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/0b7a7ba11f5e4db491107d1ce5384847 is 50, key is test_row_0/B:col10/1732416144783/Put/seqid=0 2024-11-24T02:42:26,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741931_1107 (size=12301) 2024-11-24T02:42:26,364 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/0b7a7ba11f5e4db491107d1ce5384847 2024-11-24T02:42:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/68071a6f58d34becab031c3c30a6f0a7 is 50, key is test_row_0/C:col10/1732416144783/Put/seqid=0 2024-11-24T02:42:26,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741932_1108 (size=12301) 2024-11-24T02:42:26,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416206589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416206619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416206619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416206619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416206619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:26,805 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/68071a6f58d34becab031c3c30a6f0a7 2024-11-24T02:42:26,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e 2024-11-24T02:42:26,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e, entries=150, sequenceid=561, filesize=12.0 K 2024-11-24T02:42:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/0b7a7ba11f5e4db491107d1ce5384847 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847 2024-11-24T02:42:26,828 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847, entries=150, sequenceid=561, filesize=12.0 K 2024-11-24T02:42:26,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/68071a6f58d34becab031c3c30a6f0a7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7 2024-11-24T02:42:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7, entries=150, sequenceid=561, filesize=12.0 K 2024-11-24T02:42:26,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 94fe1d236a5fa4edae1b3d10d452f90e in 990ms, sequenceid=561, compaction requested=false 2024-11-24T02:42:26,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:26,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:26,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T02:42:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T02:42:26,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T02:42:26,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-24T02:42:26,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0790 sec 2024-11-24T02:42:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:42:26,939 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:26,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T02:42:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:42:26,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:26,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:26,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:27,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:42:27,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T02:42:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:27,097 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=125.23 KB heapSize=363.39 KB 2024-11-24T02:42:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:27,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:27,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a57796ddfc3d499cba7ab2ba38cb5a66 is 50, key is test_row_0/A:col10/1732416145983/Put/seqid=0 2024-11-24T02:42:27,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741933_1109 (size=12301) 2024-11-24T02:42:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416207177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416207177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416207177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416207178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416207178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:42:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416207299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416207299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416207300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416207300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416207300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416207509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416207509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416207509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416207510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416207519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a57796ddfc3d499cba7ab2ba38cb5a66 2024-11-24T02:42:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/66081557e2f445eabe53e88bb430d8b8 is 50, key is test_row_0/B:col10/1732416145983/Put/seqid=0 2024-11-24T02:42:27,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741934_1110 (size=12301) 2024-11-24T02:42:27,552 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/66081557e2f445eabe53e88bb430d8b8 2024-11-24T02:42:27,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0b07b9287abc46a6aa351a7cdc081a8f is 50, key is test_row_0/C:col10/1732416145983/Put/seqid=0 2024-11-24T02:42:27,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741935_1111 (size=12301) 2024-11-24T02:42:27,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=593 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0b07b9287abc46a6aa351a7cdc081a8f 2024-11-24T02:42:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:42:27,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a57796ddfc3d499cba7ab2ba38cb5a66 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66 2024-11-24T02:42:27,585 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66, entries=150, sequenceid=593, filesize=12.0 K 2024-11-24T02:42:27,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/66081557e2f445eabe53e88bb430d8b8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8 2024-11-24T02:42:27,593 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8, entries=150, sequenceid=593, filesize=12.0 K 2024-11-24T02:42:27,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0b07b9287abc46a6aa351a7cdc081a8f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f 2024-11-24T02:42:27,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f, entries=150, sequenceid=593, filesize=12.0 K 2024-11-24T02:42:27,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~363.34 KB/372064, currentSize=67.09 KB/68700 for 94fe1d236a5fa4edae1b3d10d452f90e in 505ms, sequenceid=593, compaction requested=true 2024-11-24T02:42:27,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:27,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:27,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T02:42:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T02:42:27,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T02:42:27,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 660 msec 2024-11-24T02:42:27,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 668 msec 2024-11-24T02:42:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:27,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/751ba131a4a644f28b60d7941ee73e1f is 50, key is test_row_0/A:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:27,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741936_1112 (size=14741) 2024-11-24T02:42:27,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/751ba131a4a644f28b60d7941ee73e1f 2024-11-24T02:42:27,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/16b688e2c75a482baa75b2b98239df5f is 50, key is test_row_0/B:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:27,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416207838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741937_1113 (size=12301) 2024-11-24T02:42:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416207839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416207840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416207842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416207852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/16b688e2c75a482baa75b2b98239df5f 2024-11-24T02:42:27,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f4f3689ff39442f69fbe0e18b13c95f0 is 50, key is test_row_0/C:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:27,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741938_1114 (size=12301) 2024-11-24T02:42:27,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f4f3689ff39442f69fbe0e18b13c95f0 2024-11-24T02:42:27,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/751ba131a4a644f28b60d7941ee73e1f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f 2024-11-24T02:42:27,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f, entries=200, sequenceid=607, filesize=14.4 K 2024-11-24T02:42:27,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/16b688e2c75a482baa75b2b98239df5f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f 2024-11-24T02:42:27,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f, entries=150, sequenceid=607, filesize=12.0 K 2024-11-24T02:42:27,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/f4f3689ff39442f69fbe0e18b13c95f0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0 2024-11-24T02:42:27,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0, entries=150, sequenceid=607, filesize=12.0 K 2024-11-24T02:42:27,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 94fe1d236a5fa4edae1b3d10d452f90e in 106ms, sequenceid=607, compaction requested=true 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:27,928 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:27,928 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:27,929 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:27,929 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:27,929 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:27,930 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/fa15dc83537448e48cbb86de0fdcff43, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=49.0 K 2024-11-24T02:42:27,930 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:27,930 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:27,930 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:27,930 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d78cd9ae21134655818c3963272cfaf4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=51.4 K 2024-11-24T02:42:27,931 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d78cd9ae21134655818c3963272cfaf4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:27,931 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fa15dc83537448e48cbb86de0fdcff43, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:27,931 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea236bc4f6cc4bbd8c35212d7c9e9d9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732416144779 2024-11-24T02:42:27,932 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7a7ba11f5e4db491107d1ce5384847, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732416144779 2024-11-24T02:42:27,932 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a57796ddfc3d499cba7ab2ba38cb5a66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732416145982 2024-11-24T02:42:27,932 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 66081557e2f445eabe53e88bb430d8b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732416145982 2024-11-24T02:42:27,933 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751ba131a4a644f28b60d7941ee73e1f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:27,933 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b688e2c75a482baa75b2b98239df5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:27,949 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:27,949 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4006235ed60e486197b7c7fc728971ec is 50, key is test_row_0/B:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:27,960 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:27,960 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/78745202eca34edb8553a84afb074a04 is 50, key is test_row_0/A:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:27,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:42:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:27,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:27,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741939_1115 (size=13459) 2024-11-24T02:42:27,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fb8c50e1af27406db0be6365c638d79b is 50, key is test_row_0/A:col10/1732416147828/Put/seqid=0 2024-11-24T02:42:27,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416207978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416207980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:27,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416207989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:27,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:27,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416207992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416207993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741940_1116 (size=13459) 2024-11-24T02:42:28,023 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/78745202eca34edb8553a84afb074a04 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/78745202eca34edb8553a84afb074a04 2024-11-24T02:42:28,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741941_1117 (size=12301) 2024-11-24T02:42:28,033 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into 78745202eca34edb8553a84afb074a04(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:28,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fb8c50e1af27406db0be6365c638d79b 2024-11-24T02:42:28,033 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:28,033 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=12, startTime=1732416147928; duration=0sec 2024-11-24T02:42:28,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:28,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:28,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:28,037 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:28,037 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:28,049 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,049 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8025223492f74914b050986f7f0fc6bc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=49.0 K 2024-11-24T02:42:28,051 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8025223492f74914b050986f7f0fc6bc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1732416144640 2024-11-24T02:42:28,052 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68071a6f58d34becab031c3c30a6f0a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1732416144779 2024-11-24T02:42:28,053 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b07b9287abc46a6aa351a7cdc081a8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=593, earliestPutTs=1732416145982 2024-11-24T02:42:28,054 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f3689ff39442f69fbe0e18b13c95f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:28,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/355e8b038c4c4788905db2a8a2e90434 is 50, key is test_row_0/B:col10/1732416147828/Put/seqid=0 2024-11-24T02:42:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:42:28,079 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:28,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T02:42:28,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:28,082 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:28,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:28,090 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:28,091 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5acfe89bb09b4cbd95704a19a7aa07bf is 50, key is test_row_0/C:col10/1732416147144/Put/seqid=0 2024-11-24T02:42:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416208099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416208099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416208099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741942_1118 (size=12301) 2024-11-24T02:42:28,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/355e8b038c4c4788905db2a8a2e90434 2024-11-24T02:42:28,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741943_1119 (size=13459) 2024-11-24T02:42:28,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416208109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416208110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/837375361d724b44920342eb42e4ec49 is 50, key is test_row_0/C:col10/1732416147828/Put/seqid=0 2024-11-24T02:42:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741944_1120 (size=12301) 2024-11-24T02:42:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:28,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416208309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416208309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416208309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416208329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416208329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,381 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4006235ed60e486197b7c7fc728971ec as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4006235ed60e486197b7c7fc728971ec 2024-11-24T02:42:28,388 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into 4006235ed60e486197b7c7fc728971ec(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:28,388 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:28,388 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=12, startTime=1732416147928; duration=0sec 2024-11-24T02:42:28,388 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:28,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:28,388 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:28,521 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/5acfe89bb09b4cbd95704a19a7aa07bf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5acfe89bb09b4cbd95704a19a7aa07bf 2024-11-24T02:42:28,526 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into 5acfe89bb09b4cbd95704a19a7aa07bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:28,526 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:28,526 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=12, startTime=1732416147928; duration=0sec 2024-11-24T02:42:28,526 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:28,526 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:28,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/837375361d724b44920342eb42e4ec49 2024-11-24T02:42:28,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/fb8c50e1af27406db0be6365c638d79b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b 2024-11-24T02:42:28,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b, entries=150, sequenceid=626, filesize=12.0 K 2024-11-24T02:42:28,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/355e8b038c4c4788905db2a8a2e90434 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434 2024-11-24T02:42:28,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434, entries=150, sequenceid=626, filesize=12.0 K 2024-11-24T02:42:28,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/837375361d724b44920342eb42e4ec49 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49 2024-11-24T02:42:28,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49, entries=150, sequenceid=626, filesize=12.0 K 2024-11-24T02:42:28,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 94fe1d236a5fa4edae1b3d10d452f90e in 640ms, sequenceid=626, compaction requested=false 2024-11-24T02:42:28,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:28,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:28,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/64fba7dd18b24b67b79235b791f075d7 is 50, key is test_row_0/A:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:28,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741945_1121 (size=12301) 2024-11-24T02:42:28,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/64fba7dd18b24b67b79235b791f075d7 2024-11-24T02:42:28,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1663ded9798d4c6d9118b4d902066fb6 is 50, key is test_row_0/B:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416208661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416208663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416208665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416208668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416208669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:28,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:28,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741946_1122 (size=12301) 2024-11-24T02:42:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:28,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416208778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416208779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416208779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416208779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416208779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:28,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:28,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:28,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416208991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416208991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416208991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416208991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416208991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1663ded9798d4c6d9118b4d902066fb6 2024-11-24T02:42:29,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/cd6d0a2ae566424eb1577d5374bcbf68 is 50, key is test_row_0/C:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:29,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741947_1123 (size=12301) 2024-11-24T02:42:29,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=645 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/cd6d0a2ae566424eb1577d5374bcbf68 2024-11-24T02:42:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/64fba7dd18b24b67b79235b791f075d7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7 2024-11-24T02:42:29,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7, entries=150, sequenceid=645, filesize=12.0 K 2024-11-24T02:42:29,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/1663ded9798d4c6d9118b4d902066fb6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6 2024-11-24T02:42:29,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6, entries=150, sequenceid=645, filesize=12.0 K 2024-11-24T02:42:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/cd6d0a2ae566424eb1577d5374bcbf68 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68 2024-11-24T02:42:29,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68, entries=150, sequenceid=645, filesize=12.0 K 2024-11-24T02:42:29,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 94fe1d236a5fa4edae1b3d10d452f90e in 555ms, sequenceid=645, compaction requested=true 2024-11-24T02:42:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:29,181 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:29,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:29,182 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:29,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/B is initiating minor compaction (all files) 2024-11-24T02:42:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94fe1d236a5fa4edae1b3d10d452f90e:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:29,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:29,183 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/B in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,183 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4006235ed60e486197b7c7fc728971ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=37.2 K 2024-11-24T02:42:29,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:29,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/A is initiating minor compaction (all files) 2024-11-24T02:42:29,186 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/A in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,186 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/78745202eca34edb8553a84afb074a04, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=37.2 K 2024-11-24T02:42:29,187 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78745202eca34edb8553a84afb074a04, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:29,187 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4006235ed60e486197b7c7fc728971ec, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:29,187 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8c50e1af27406db0be6365c638d79b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732416147828 2024-11-24T02:42:29,187 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 355e8b038c4c4788905db2a8a2e90434, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732416147828 2024-11-24T02:42:29,188 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64fba7dd18b24b67b79235b791f075d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1732416147991 2024-11-24T02:42:29,188 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1663ded9798d4c6d9118b4d902066fb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1732416147991 2024-11-24T02:42:29,206 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:29,206 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/dedbfd83f2f94794ab092109177d3eb3 is 50, key is test_row_0/B:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:29,209 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#A#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:29,209 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/f00efe9040a9459299d7d97fe6ce3a14 is 50, key is test_row_0/A:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741948_1124 (size=13561) 2024-11-24T02:42:29,230 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/f00efe9040a9459299d7d97fe6ce3a14 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f00efe9040a9459299d7d97fe6ce3a14 2024-11-24T02:42:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741949_1125 (size=13561) 2024-11-24T02:42:29,262 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/A of 94fe1d236a5fa4edae1b3d10d452f90e into f00efe9040a9459299d7d97fe6ce3a14(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:29,262 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:29,262 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/A, priority=13, startTime=1732416149181; duration=0sec 2024-11-24T02:42:29,262 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:29,262 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:A 2024-11-24T02:42:29,262 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:29,265 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:29,265 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): 94fe1d236a5fa4edae1b3d10d452f90e/C is initiating minor compaction (all files) 2024-11-24T02:42:29,265 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94fe1d236a5fa4edae1b3d10d452f90e/C in TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,265 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5acfe89bb09b4cbd95704a19a7aa07bf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp, totalSize=37.2 K 2024-11-24T02:42:29,266 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5acfe89bb09b4cbd95704a19a7aa07bf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1732416147144 2024-11-24T02:42:29,267 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837375361d724b44920342eb42e4ec49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=626, earliestPutTs=1732416147828 2024-11-24T02:42:29,268 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6d0a2ae566424eb1577d5374bcbf68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=645, earliestPutTs=1732416147991 2024-11-24T02:42:29,274 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/dedbfd83f2f94794ab092109177d3eb3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/dedbfd83f2f94794ab092109177d3eb3 2024-11-24T02:42:29,284 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/B of 94fe1d236a5fa4edae1b3d10d452f90e into dedbfd83f2f94794ab092109177d3eb3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:29,284 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:29,284 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/B, priority=13, startTime=1732416149181; duration=0sec 2024-11-24T02:42:29,284 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:29,284 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:B 2024-11-24T02:42:29,289 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94fe1d236a5fa4edae1b3d10d452f90e#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:29,289 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d5577638648c48418ab42e4fac3167e7 is 50, key is test_row_0/C:col10/1732416148622/Put/seqid=0 2024-11-24T02:42:29,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741950_1126 (size=13561) 2024-11-24T02:42:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:29,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:42:29,309 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/d5577638648c48418ab42e4fac3167e7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5577638648c48418ab42e4fac3167e7 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,333 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94fe1d236a5fa4edae1b3d10d452f90e/C of 94fe1d236a5fa4edae1b3d10d452f90e into d5577638648c48418ab42e4fac3167e7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:29,333 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:29,333 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., storeName=94fe1d236a5fa4edae1b3d10d452f90e/C, priority=13, startTime=1732416149183; duration=0sec 2024-11-24T02:42:29,334 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:29,334 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94fe1d236a5fa4edae1b3d10d452f90e:C 2024-11-24T02:42:29,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/47ec141c15d549d7919018ca9f1174b7 is 50, key is test_row_0/A:col10/1732416148662/Put/seqid=0 2024-11-24T02:42:29,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416209333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416209335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416209339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416209340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416209341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741951_1127 (size=14741) 2024-11-24T02:42:29,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416209460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416209460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416209468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416209468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416209463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49790 deadline: 1732416209679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49826 deadline: 1732416209679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49810 deadline: 1732416209679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1732416209679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1732416209689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:42:29,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/47ec141c15d549d7919018ca9f1174b7 2024-11-24T02:42:29,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/12f545cb6e2e4ba8b497a08b4efd9e62 is 50, key is test_row_0/B:col10/1732416148662/Put/seqid=0 2024-11-24T02:42:29,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741952_1128 (size=12301) 2024-11-24T02:42:29,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/12f545cb6e2e4ba8b497a08b4efd9e62 2024-11-24T02:42:29,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 is 50, key is test_row_0/C:col10/1732416148662/Put/seqid=0 2024-11-24T02:42:29,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741953_1129 (size=12301) 2024-11-24T02:42:29,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 2024-11-24T02:42:29,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/47ec141c15d549d7919018ca9f1174b7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/47ec141c15d549d7919018ca9f1174b7 2024-11-24T02:42:29,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/47ec141c15d549d7919018ca9f1174b7, entries=200, sequenceid=665, filesize=14.4 K 2024-11-24T02:42:29,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/12f545cb6e2e4ba8b497a08b4efd9e62 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/12f545cb6e2e4ba8b497a08b4efd9e62 2024-11-24T02:42:29,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/12f545cb6e2e4ba8b497a08b4efd9e62, entries=150, sequenceid=665, filesize=12.0 K 2024-11-24T02:42:29,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 2024-11-24T02:42:29,882 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T02:42:29,882 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:29,883 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T02:42:29,883 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:29,884 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T02:42:29,884 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:29,885 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T02:42:29,885 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:29,888 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8f4c24672cf04ad9a00c8d9fdfb31ee8, entries=150, sequenceid=665, filesize=12.0 K 2024-11-24T02:42:29,888 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:29,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:29,889 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:29,889 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:29,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:29,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 94fe1d236a5fa4edae1b3d10d452f90e in 581ms, sequenceid=665, compaction requested=false 2024-11-24T02:42:29,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:29,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:29,937 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:29,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/bc9c074d67b8463ea9b3721559a5afe9 is 50, key is test_row_0/A:col10/1732416149330/Put/seqid=0 2024-11-24T02:42:29,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741954_1130 (size=12301) 2024-11-24T02:42:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:30,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. as already flushing 2024-11-24T02:42:30,003 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T02:42:30,003 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T02:42:30,003 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:30,003 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:30,003 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T02:42:30,003 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:30,003 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:30,003 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:30,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:30,006 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T02:42:30,006 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:30,006 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,006 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:30,007 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T02:42:30,007 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:30,007 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,007 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:30,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:30,346 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/bc9c074d67b8463ea9b3721559a5afe9 2024-11-24T02:42:30,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4971d38e756144da81fd1f475e3ee564 is 50, key is test_row_0/B:col10/1732416149330/Put/seqid=0 2024-11-24T02:42:30,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741955_1131 (size=12301) 2024-11-24T02:42:30,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4971d38e756144da81fd1f475e3ee564 2024-11-24T02:42:30,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/a6f26a6a6f1f4259827bc1ecca4081c1 is 50, key is test_row_0/C:col10/1732416149330/Put/seqid=0 2024-11-24T02:42:30,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741956_1132 (size=12301) 2024-11-24T02:42:31,171 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=680 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/a6f26a6a6f1f4259827bc1ecca4081c1 2024-11-24T02:42:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/bc9c074d67b8463ea9b3721559a5afe9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/bc9c074d67b8463ea9b3721559a5afe9 2024-11-24T02:42:31,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/bc9c074d67b8463ea9b3721559a5afe9, entries=150, sequenceid=680, filesize=12.0 K 2024-11-24T02:42:31,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/4971d38e756144da81fd1f475e3ee564 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4971d38e756144da81fd1f475e3ee564 2024-11-24T02:42:31,185 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4971d38e756144da81fd1f475e3ee564, entries=150, sequenceid=680, filesize=12.0 K 2024-11-24T02:42:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/a6f26a6a6f1f4259827bc1ecca4081c1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a6f26a6a6f1f4259827bc1ecca4081c1 2024-11-24T02:42:31,191 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a6f26a6a6f1f4259827bc1ecca4081c1, entries=150, sequenceid=680, filesize=12.0 K 2024-11-24T02:42:31,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for 94fe1d236a5fa4edae1b3d10d452f90e in 1255ms, sequenceid=680, compaction requested=true 2024-11-24T02:42:31,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 94fe1d236a5fa4edae1b3d10d452f90e: 2024-11-24T02:42:31,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:31,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T02:42:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T02:42:31,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T02:42:31,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1100 sec 2024-11-24T02:42:31,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 3.1150 sec 2024-11-24T02:42:31,288 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:42:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:42:32,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:32,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:42:32,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-24T02:42:32,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-24T02:42:32,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-24T02:42:32,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2391 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2396 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3238 rows 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1079 2024-11-24T02:42:32,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3230 rows 2024-11-24T02:42:32,251 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:42:32,251 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:42:32,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:32,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:32,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:32,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:42:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:32,264 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416152264"}]},"ts":"1732416152264"} 2024-11-24T02:42:32,266 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:42:32,266 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:42:32,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:42:32,274 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, UNASSIGN}] 2024-11-24T02:42:32,275 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, UNASSIGN 2024-11-24T02:42:32,277 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=94fe1d236a5fa4edae1b3d10d452f90e, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:32,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, UNASSIGN because future has completed 2024-11-24T02:42:32,281 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:42:32,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:32,437 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:32,437 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:42:32,438 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 94fe1d236a5fa4edae1b3d10d452f90e, disabling compactions & flushes 2024-11-24T02:42:32,438 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:32,438 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:32,438 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. after waiting 0 ms 2024-11-24T02:42:32,438 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:32,438 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 94fe1d236a5fa4edae1b3d10d452f90e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=A 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=B 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 94fe1d236a5fa4edae1b3d10d452f90e, store=C 2024-11-24T02:42:32,439 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:32,443 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a585b54c74d646bab86c71dc97eb379a is 50, key is test_row_0/A:col10/1732416150003/Put/seqid=0 2024-11-24T02:42:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741957_1133 (size=12301) 2024-11-24T02:42:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:32,848 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a585b54c74d646bab86c71dc97eb379a 2024-11-24T02:42:32,860 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/83257796294f4d719973a1526fa9cb9d is 50, key is test_row_0/B:col10/1732416150003/Put/seqid=0 2024-11-24T02:42:32,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741958_1134 (size=12301) 2024-11-24T02:42:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:33,266 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/83257796294f4d719973a1526fa9cb9d 2024-11-24T02:42:33,276 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d1048698a8b4c4eabc9ddff39f5d5fc is 50, key is test_row_0/C:col10/1732416150003/Put/seqid=0 2024-11-24T02:42:33,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741959_1135 (size=12301) 2024-11-24T02:42:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:33,682 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d1048698a8b4c4eabc9ddff39f5d5fc 2024-11-24T02:42:33,688 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/A/a585b54c74d646bab86c71dc97eb379a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a585b54c74d646bab86c71dc97eb379a 2024-11-24T02:42:33,694 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a585b54c74d646bab86c71dc97eb379a, entries=150, sequenceid=688, filesize=12.0 K 2024-11-24T02:42:33,695 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/B/83257796294f4d719973a1526fa9cb9d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/83257796294f4d719973a1526fa9cb9d 2024-11-24T02:42:33,700 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/83257796294f4d719973a1526fa9cb9d, entries=150, sequenceid=688, filesize=12.0 K 2024-11-24T02:42:33,701 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/.tmp/C/0d1048698a8b4c4eabc9ddff39f5d5fc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d1048698a8b4c4eabc9ddff39f5d5fc 2024-11-24T02:42:33,705 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d1048698a8b4c4eabc9ddff39f5d5fc, entries=150, sequenceid=688, filesize=12.0 K 2024-11-24T02:42:33,706 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 94fe1d236a5fa4edae1b3d10d452f90e in 1268ms, sequenceid=688, compaction requested=true 2024-11-24T02:42:33,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/71ec500084d041be93484e0ed0121730, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/8f1e2bbe41b549f0846cdca4cc4c5fc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c039fb84848b4cbe98e64a868af82c6b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/7777c580f3c6467d8358f9fdd14d743d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ad48256046c847a49b4e551b71468a82, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c558257adea542cd90fb829e657dd8dd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/0fef887887c045999109d2a063bb06ca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/81889dad0fa741238b0fe3e45db484b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d78cd9ae21134655818c3963272cfaf4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/78745202eca34edb8553a84afb074a04, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7] to archive 2024-11-24T02:42:33,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:42:33,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/5cfec8a97a24495ca9541b594c94ec23 2024-11-24T02:42:33,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/76fd934466ac40798071e9d446ff0689 2024-11-24T02:42:33,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cfe788e3148942f0902025d803550d45 2024-11-24T02:42:33,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/71ec500084d041be93484e0ed0121730 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/71ec500084d041be93484e0ed0121730 2024-11-24T02:42:33,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f389e6bbaa954bcd8f1133fd7c693e81 2024-11-24T02:42:33,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/8f1e2bbe41b549f0846cdca4cc4c5fc4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/8f1e2bbe41b549f0846cdca4cc4c5fc4 2024-11-24T02:42:33,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6e3b97bdfb724d13b2a8de8bc64b5065 2024-11-24T02:42:33,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cedbd4c919b44a37b64b9952770ca95d 2024-11-24T02:42:33,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d1e7eef88a2b4a31a6514c7a6d3514ea 2024-11-24T02:42:33,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c039fb84848b4cbe98e64a868af82c6b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c039fb84848b4cbe98e64a868af82c6b 2024-11-24T02:42:33,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d2723a62f67643e28ce8cdfe9f1b2544 2024-11-24T02:42:33,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/431eac284d084145bd46120ff24d9f8e 2024-11-24T02:42:33,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/7777c580f3c6467d8358f9fdd14d743d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/7777c580f3c6467d8358f9fdd14d743d 2024-11-24T02:42:33,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/cf21e30da4424ea8a6f221d30a9afc39 2024-11-24T02:42:33,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/65dae36e9af243928a9faae714c30656 2024-11-24T02:42:33,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/73471c547e6141328610105b06f2d00d 2024-11-24T02:42:33,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ad48256046c847a49b4e551b71468a82 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ad48256046c847a49b4e551b71468a82 2024-11-24T02:42:33,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/9e4a5cd0d20642829302696ed7ec1b27 2024-11-24T02:42:33,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/2cc5a69c2aa64d5fa90da4b621f82d46 2024-11-24T02:42:33,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c558257adea542cd90fb829e657dd8dd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c558257adea542cd90fb829e657dd8dd 2024-11-24T02:42:33,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/175305a1d986490fb93a9d4d8ed3231b 2024-11-24T02:42:33,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/3740bc93d5b14f79880e95d04b239009 2024-11-24T02:42:33,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/471c1b8537614c3aa77b0ff5f9c628b1 2024-11-24T02:42:33,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/0fef887887c045999109d2a063bb06ca to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/0fef887887c045999109d2a063bb06ca 2024-11-24T02:42:33,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/6b349979e12546c482c27c753bfa2929 2024-11-24T02:42:33,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/81889dad0fa741238b0fe3e45db484b9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/81889dad0fa741238b0fe3e45db484b9 2024-11-24T02:42:33,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ef4ea1d740ae4771aff539c2955a7c4f 2024-11-24T02:42:33,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fd9ea872868d4298927336d18e6fe481 2024-11-24T02:42:33,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/c0048ec393d74e509ae9778193e02e5d 2024-11-24T02:42:33,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/17946d34a4ff43e6b211543b15a4654d 2024-11-24T02:42:33,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d78cd9ae21134655818c3963272cfaf4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/d78cd9ae21134655818c3963272cfaf4 2024-11-24T02:42:33,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/ea236bc4f6cc4bbd8c35212d7c9e9d9e 2024-11-24T02:42:33,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a57796ddfc3d499cba7ab2ba38cb5a66 2024-11-24T02:42:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/751ba131a4a644f28b60d7941ee73e1f 2024-11-24T02:42:33,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/78745202eca34edb8553a84afb074a04 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/78745202eca34edb8553a84afb074a04 2024-11-24T02:42:33,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/fb8c50e1af27406db0be6365c638d79b 2024-11-24T02:42:33,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/64fba7dd18b24b67b79235b791f075d7 2024-11-24T02:42:33,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a23fc494d7114a97980630fbc36306b7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5c5cda0a1c4c48debe2ac9f2759e4143, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/6c5287f675d940648d516441f0a0ef91, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/8fecb3fd0da94257a3ff70880193d14c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f5446bf2d5341f6a14dd53be7fc4165, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e8774e943dc24e259de55eb2ecae5810, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4b074f64bdcc4dce9b933fbe7a4bb2fc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/072cb262b45441a4ac1a341a4cae94c2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/fa15dc83537448e48cbb86de0fdcff43, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4006235ed60e486197b7c7fc728971ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6] to archive 2024-11-24T02:42:33,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:42:33,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/26d94f61215348d8ab3531f4b280554f 2024-11-24T02:42:33,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/86c2b77ffc9440848fa1f392bb6164bc 2024-11-24T02:42:33,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a23fc494d7114a97980630fbc36306b7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a23fc494d7114a97980630fbc36306b7 2024-11-24T02:42:33,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9ce16832909a4dbdb18db44be4f548c2 2024-11-24T02:42:33,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/409dfe3c433d41ba95fd20d67d039fe4 2024-11-24T02:42:33,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5c5cda0a1c4c48debe2ac9f2759e4143 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5c5cda0a1c4c48debe2ac9f2759e4143 2024-11-24T02:42:33,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/90a5652e5b3a41678fb2876462b27b89 2024-11-24T02:42:33,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a36a3113da484704851fbbb3821ef334 2024-11-24T02:42:33,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/6c5287f675d940648d516441f0a0ef91 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/6c5287f675d940648d516441f0a0ef91 2024-11-24T02:42:33,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e784256f5d2a4daeb656ce9ebe1c5d73 2024-11-24T02:42:33,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a49c146ff32d4d5e8afde41bfd9554e8 2024-11-24T02:42:33,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f684591f84f84385a7366baefc6ab830 2024-11-24T02:42:33,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/8fecb3fd0da94257a3ff70880193d14c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/8fecb3fd0da94257a3ff70880193d14c 2024-11-24T02:42:33,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1e77df7126c5410a98cf5b6134f5748d 2024-11-24T02:42:33,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/50bffedeab1c4aa98d0902e190b75f27 2024-11-24T02:42:33,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f5446bf2d5341f6a14dd53be7fc4165 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f5446bf2d5341f6a14dd53be7fc4165 2024-11-24T02:42:33,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/78b42699eefc4dbf881c39f986958229 2024-11-24T02:42:33,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/9f6920300bcd4000ac6bfe42ab7b290d 2024-11-24T02:42:33,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/70fb6770b6f04409939b680a16240a96 2024-11-24T02:42:33,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e8774e943dc24e259de55eb2ecae5810 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/e8774e943dc24e259de55eb2ecae5810 2024-11-24T02:42:33,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/140f98b990e94acbbacf7cf5d377dbd0 2024-11-24T02:42:33,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/c6e3ac3231fc4210b04c59ba164b0275 2024-11-24T02:42:33,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4b074f64bdcc4dce9b933fbe7a4bb2fc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4b074f64bdcc4dce9b933fbe7a4bb2fc 2024-11-24T02:42:33,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/f8e2a116d81543938a70b7cc08c7b2a7 2024-11-24T02:42:33,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/b3f51d76a7a348de968460d974946a23 2024-11-24T02:42:33,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/072cb262b45441a4ac1a341a4cae94c2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/072cb262b45441a4ac1a341a4cae94c2 2024-11-24T02:42:33,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/58f7bd22f76d434382c938113c399b92 2024-11-24T02:42:33,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/a988d5a26d7c4d8bbce8ade021126b86 2024-11-24T02:42:33,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/41678e591a744345b38f784ea7ed6171 2024-11-24T02:42:33,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/fa15dc83537448e48cbb86de0fdcff43 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/fa15dc83537448e48cbb86de0fdcff43 2024-11-24T02:42:33,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/5a437567817e49809a72018b494ab596 2024-11-24T02:42:33,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/0b7a7ba11f5e4db491107d1ce5384847 2024-11-24T02:42:33,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/66081557e2f445eabe53e88bb430d8b8 2024-11-24T02:42:33,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4006235ed60e486197b7c7fc728971ec to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4006235ed60e486197b7c7fc728971ec 2024-11-24T02:42:33,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/16b688e2c75a482baa75b2b98239df5f 2024-11-24T02:42:33,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/355e8b038c4c4788905db2a8a2e90434 2024-11-24T02:42:33,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/1663ded9798d4c6d9118b4d902066fb6 2024-11-24T02:42:33,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b973a73992bf4732a04bc26f7349fb0a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c5c94bc34d86417da1103d2c37014aaf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c057fd09b83417a8ace093aa8755c33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/ed4b871f8e614b55983829068ddcf0a1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b8cdb5139a824f3290b992502a2cd689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a5fef007890a4beb8b24c3963d08d5d8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/aac4ea354b064670bbfba50fb3729955, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c7ad6cdb8d9643bc84e8c0e86410aaa7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8025223492f74914b050986f7f0fc6bc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5acfe89bb09b4cbd95704a19a7aa07bf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68] to archive 2024-11-24T02:42:33,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:42:33,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/daf87161e77448c1945d1466bcbb30f5 2024-11-24T02:42:33,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5317eca6bef74c5dbfa7e31e8a6a20a9 2024-11-24T02:42:33,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b973a73992bf4732a04bc26f7349fb0a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b973a73992bf4732a04bc26f7349fb0a 2024-11-24T02:42:33,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/021640de4e0a47619be8cdc5905d30e0 2024-11-24T02:42:33,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5108a5df916f4008a672a535f29b50dd 2024-11-24T02:42:33,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/da365121aa854463b163da0dd8337b01 2024-11-24T02:42:33,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c5c94bc34d86417da1103d2c37014aaf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c5c94bc34d86417da1103d2c37014aaf 2024-11-24T02:42:33,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/fa59447681014f9c9b10cd1a7f24e41b 2024-11-24T02:42:33,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d46724c148ff4f2489af204f19de97d3 2024-11-24T02:42:33,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c057fd09b83417a8ace093aa8755c33 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c057fd09b83417a8ace093aa8755c33 2024-11-24T02:42:33,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f9127307e81b4ef28173476d9c2a5361 2024-11-24T02:42:33,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/de40cf8994624f12add42824665b1a51 2024-11-24T02:42:33,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/ed4b871f8e614b55983829068ddcf0a1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/ed4b871f8e614b55983829068ddcf0a1 2024-11-24T02:42:33,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/58e41adf706943a29c186c953a34ee50 2024-11-24T02:42:33,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/db55856c9e484837ae81599681107fc7 2024-11-24T02:42:33,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b8cdb5139a824f3290b992502a2cd689 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b8cdb5139a824f3290b992502a2cd689 2024-11-24T02:42:33,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5c080e68bd9a4ed2b78eeabb696311fb 2024-11-24T02:42:33,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5ac1a856be8d48b9afb21804d01b02e6 2024-11-24T02:42:33,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/83bcbb3ee2c84fb7b48e1f3ec6a8d806 2024-11-24T02:42:33,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a5fef007890a4beb8b24c3963d08d5d8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a5fef007890a4beb8b24c3963d08d5d8 2024-11-24T02:42:33,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d2a56303b054c40a54bab8364740c18 2024-11-24T02:42:33,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5037486c15a470d92ed7519c33f1145 2024-11-24T02:42:33,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/aac4ea354b064670bbfba50fb3729955 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/aac4ea354b064670bbfba50fb3729955 2024-11-24T02:42:33,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/481aec059bda40eaac9f9696e366a766 2024-11-24T02:42:33,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/01585a4448664295b4dcf9591ed974d7 2024-11-24T02:42:33,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c7ad6cdb8d9643bc84e8c0e86410aaa7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/c7ad6cdb8d9643bc84e8c0e86410aaa7 2024-11-24T02:42:33,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/171efe908d5a4107be7115c6db4f2c14 2024-11-24T02:42:33,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/b22f44cb33b74e2ba67710efe458caf8 2024-11-24T02:42:33,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/6824c0bffa624e938a2fce819a69a257 2024-11-24T02:42:33,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8025223492f74914b050986f7f0fc6bc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8025223492f74914b050986f7f0fc6bc 2024-11-24T02:42:33,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/778cb49cf20a4371a0c0b51c8292907d 2024-11-24T02:42:33,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/68071a6f58d34becab031c3c30a6f0a7 2024-11-24T02:42:33,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0b07b9287abc46a6aa351a7cdc081a8f 2024-11-24T02:42:33,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5acfe89bb09b4cbd95704a19a7aa07bf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/5acfe89bb09b4cbd95704a19a7aa07bf 2024-11-24T02:42:33,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/f4f3689ff39442f69fbe0e18b13c95f0 2024-11-24T02:42:33,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/837375361d724b44920342eb42e4ec49 2024-11-24T02:42:33,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/cd6d0a2ae566424eb1577d5374bcbf68 2024-11-24T02:42:33,886 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/recovered.edits/691.seqid, newMaxSeqId=691, maxSeqId=1 2024-11-24T02:42:33,887 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. 2024-11-24T02:42:33,887 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 94fe1d236a5fa4edae1b3d10d452f90e: Waiting for close lock at 1732416152438Running coprocessor pre-close hooks at 1732416152438Disabling compacts and flushes for region at 1732416152438Disabling writes for close at 1732416152438Obtaining lock to block concurrent updates at 1732416152438Preparing flush snapshotting stores in 94fe1d236a5fa4edae1b3d10d452f90e at 1732416152438Finished memstore snapshotting TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732416152439 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e. at 1732416152440 (+1 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/A: creating writer at 1732416152440Flushing 94fe1d236a5fa4edae1b3d10d452f90e/A: appending metadata at 1732416152443 (+3 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/A: closing flushed file at 1732416152443Flushing 94fe1d236a5fa4edae1b3d10d452f90e/B: creating writer at 1732416152856 (+413 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/B: appending metadata at 1732416152860 (+4 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/B: closing flushed file at 1732416152860Flushing 94fe1d236a5fa4edae1b3d10d452f90e/C: creating writer at 1732416153272 (+412 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/C: appending metadata at 1732416153275 (+3 ms)Flushing 94fe1d236a5fa4edae1b3d10d452f90e/C: closing flushed file at 1732416153275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d25e146: reopening flushed file at 1732416153687 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@348c313f: reopening flushed file at 1732416153694 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@342aca0d: reopening flushed file at 1732416153700 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 94fe1d236a5fa4edae1b3d10d452f90e in 1268ms, sequenceid=688, compaction requested=true at 1732416153706 (+6 ms)Writing region close event to WAL at 1732416153882 (+176 ms)Running coprocessor post-close hooks at 1732416153887 (+5 ms)Closed at 1732416153887 2024-11-24T02:42:33,890 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:33,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=94fe1d236a5fa4edae1b3d10d452f90e, regionState=CLOSED 2024-11-24T02:42:33,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:42:33,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T02:42:33,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 94fe1d236a5fa4edae1b3d10d452f90e, server=7c69a60bd8f6,34251,1732416122955 in 1.6160 sec 2024-11-24T02:42:33,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T02:42:33,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94fe1d236a5fa4edae1b3d10d452f90e, UNASSIGN in 1.6240 sec 2024-11-24T02:42:33,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T02:42:33,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6340 sec 2024-11-24T02:42:33,907 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416153906"}]},"ts":"1732416153906"} 2024-11-24T02:42:33,908 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:42:33,908 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:42:33,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6530 sec 2024-11-24T02:42:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:42:34,410 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:34,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:42:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:42:34,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,428 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:34,432 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/recovered.edits] 2024-11-24T02:42:34,434 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/47ec141c15d549d7919018ca9f1174b7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/47ec141c15d549d7919018ca9f1174b7 2024-11-24T02:42:34,436 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a585b54c74d646bab86c71dc97eb379a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/a585b54c74d646bab86c71dc97eb379a 2024-11-24T02:42:34,439 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/bc9c074d67b8463ea9b3721559a5afe9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/bc9c074d67b8463ea9b3721559a5afe9 2024-11-24T02:42:34,441 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f00efe9040a9459299d7d97fe6ce3a14 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/A/f00efe9040a9459299d7d97fe6ce3a14 2024-11-24T02:42:34,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/12f545cb6e2e4ba8b497a08b4efd9e62 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/12f545cb6e2e4ba8b497a08b4efd9e62 2024-11-24T02:42:34,449 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4971d38e756144da81fd1f475e3ee564 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/4971d38e756144da81fd1f475e3ee564 2024-11-24T02:42:34,453 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/83257796294f4d719973a1526fa9cb9d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/83257796294f4d719973a1526fa9cb9d 2024-11-24T02:42:34,456 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/dedbfd83f2f94794ab092109177d3eb3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/B/dedbfd83f2f94794ab092109177d3eb3 2024-11-24T02:42:34,459 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d1048698a8b4c4eabc9ddff39f5d5fc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/0d1048698a8b4c4eabc9ddff39f5d5fc 2024-11-24T02:42:34,461 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/8f4c24672cf04ad9a00c8d9fdfb31ee8 2024-11-24T02:42:34,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a6f26a6a6f1f4259827bc1ecca4081c1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/a6f26a6a6f1f4259827bc1ecca4081c1 2024-11-24T02:42:34,464 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5577638648c48418ab42e4fac3167e7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/C/d5577638648c48418ab42e4fac3167e7 2024-11-24T02:42:34,467 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/recovered.edits/691.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e/recovered.edits/691.seqid 2024-11-24T02:42:34,468 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/94fe1d236a5fa4edae1b3d10d452f90e 2024-11-24T02:42:34,468 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:42:34,474 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T02:42:34,486 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:42:34,531 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:42:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:42:34,534 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,534 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:42:34,535 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416154534"}]},"ts":"9223372036854775807"} 2024-11-24T02:42:34,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:42:34,542 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94fe1d236a5fa4edae1b3d10d452f90e, NAME => 'TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:42:34,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:42:34,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416154543"}]},"ts":"9223372036854775807"} 2024-11-24T02:42:34,551 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:42:34,552 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 137 msec 2024-11-24T02:42:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:42:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:42:34,740 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:34,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_693650994_22 at /127.0.0.1:60490 [Waiting for operation #374] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1130878907_22 at /127.0.0.1:60374 [Waiting for operation #389] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7c69a60bd8f6:34251-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 237) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10690 (was 10548) - AvailableMemoryMB LEAK? - 2024-11-24T02:42:34,760 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=10690 2024-11-24T02:42:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:42:34,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:42:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:34,764 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:42:34,765 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:34,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-24T02:42:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:34,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:42:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741960_1136 (size=1061) 2024-11-24T02:42:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:35,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:42:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741961_1137 (size=53) 2024-11-24T02:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:35,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:35,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a7378d33aaa14aba026a2abf2aaf9320, disabling compactions & flushes 2024-11-24T02:42:35,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. after waiting 0 ms 2024-11-24T02:42:35,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,589 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7378d33aaa14aba026a2abf2aaf9320: Waiting for close lock at 1732416155588Disabling compacts and flushes for region at 1732416155588Disabling writes for close at 1732416155588Writing region close event to WAL at 1732416155588Closed at 1732416155588 2024-11-24T02:42:35,590 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:42:35,590 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416155590"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416155590"}]},"ts":"1732416155590"} 2024-11-24T02:42:35,593 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:42:35,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:42:35,596 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416155596"}]},"ts":"1732416155596"} 2024-11-24T02:42:35,599 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:42:35,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, ASSIGN}] 2024-11-24T02:42:35,602 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, ASSIGN 2024-11-24T02:42:35,603 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:42:35,754 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:35,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, ASSIGN because future has completed 2024-11-24T02:42:35,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:35,916 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,916 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:42:35,916 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,917 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:35,917 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,917 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,918 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,920 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:35,921 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName A 2024-11-24T02:42:35,921 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:35,922 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:35,922 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,923 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:35,924 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName B 2024-11-24T02:42:35,924 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:35,924 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:35,924 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,925 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:35,926 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName C 2024-11-24T02:42:35,926 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:35,926 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:35,926 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,926 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,927 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,928 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,930 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:42:35,932 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,934 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:42:35,934 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a7378d33aaa14aba026a2abf2aaf9320; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63826552, jitterRate=-0.04891026020050049}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:42:35,934 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:35,935 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a7378d33aaa14aba026a2abf2aaf9320: Running coprocessor pre-open hook at 1732416155917Writing region info on filesystem at 1732416155917Initializing all the Stores at 1732416155918 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416155918Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416155918Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416155918Cleaning up temporary data from old regions at 1732416155929 (+11 ms)Running coprocessor post-open hooks at 1732416155934 (+5 ms)Region opened successfully at 1732416155935 (+1 ms) 2024-11-24T02:42:35,936 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., pid=32, masterSystemTime=1732416155910 2024-11-24T02:42:35,938 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,938 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:35,939 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:35,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:42:35,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T02:42:35,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 in 184 msec 2024-11-24T02:42:35,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T02:42:35,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, ASSIGN in 344 msec 2024-11-24T02:42:35,946 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:42:35,946 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416155946"}]},"ts":"1732416155946"} 2024-11-24T02:42:35,948 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:42:35,949 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:42:35,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-24T02:42:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:42:36,909 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:36,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:36,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:36,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:36,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23746f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:36,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:36,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28fc2100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:36,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:36,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:36,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:36,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:36,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:42:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:42:36,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53dfebc4 2024-11-24T02:42:36,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:42:36,928 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:42:36,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:42:36,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:42:36,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:42:36,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741962_1138 (size=1097) 2024-11-24T02:42:37,370 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T02:42:37,371 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T02:42:37,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:42:37,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE}] 2024-11-24T02:42:37,387 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE 2024-11-24T02:42:37,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:37,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE because future has completed 2024-11-24T02:42:37,390 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:42:37,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:37,544 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:42:37,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing a7378d33aaa14aba026a2abf2aaf9320, disabling compactions & flushes 2024-11-24T02:42:37,544 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. after waiting 0 ms 2024-11-24T02:42:37,545 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,551 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:42:37,551 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,551 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for a7378d33aaa14aba026a2abf2aaf9320: Waiting for close lock at 1732416157544Running coprocessor pre-close hooks at 1732416157544Disabling compacts and flushes for region at 1732416157544Disabling writes for close at 1732416157545 (+1 ms)Writing region close event to WAL at 1732416157545Running coprocessor post-close hooks at 1732416157551 (+6 ms)Closed at 1732416157551 2024-11-24T02:42:37,552 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: a7378d33aaa14aba026a2abf2aaf9320 to self. 2024-11-24T02:42:37,554 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,555 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=CLOSED 2024-11-24T02:42:37,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:42:37,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T02:42:37,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 in 168 msec 2024-11-24T02:42:37,561 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=true 2024-11-24T02:42:37,711 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:37,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE because future has completed 2024-11-24T02:42:37,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:42:37,891 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,892 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:42:37,892 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,892 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:42:37,892 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,892 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,898 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,900 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:37,907 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName A 2024-11-24T02:42:37,910 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:37,910 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:37,911 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,912 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:37,912 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName B 2024-11-24T02:42:37,912 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:37,913 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:37,913 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,913 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:42:37,914 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7378d33aaa14aba026a2abf2aaf9320 columnFamilyName C 2024-11-24T02:42:37,914 DEBUG [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:37,915 INFO [StoreOpener-a7378d33aaa14aba026a2abf2aaf9320-1 {}] regionserver.HStore(327): Store=a7378d33aaa14aba026a2abf2aaf9320/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:42:37,915 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,915 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,916 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,917 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,919 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,919 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,922 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:42:37,931 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,933 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened a7378d33aaa14aba026a2abf2aaf9320; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65068869, jitterRate=-0.03039829432964325}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:42:37,933 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:37,935 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for a7378d33aaa14aba026a2abf2aaf9320: Running coprocessor pre-open hook at 1732416157893Writing region info on filesystem at 1732416157893Initializing all the Stores at 1732416157894 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416157894Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416157896 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416157896Cleaning up temporary data from old regions at 1732416157919 (+23 ms)Running coprocessor post-open hooks at 1732416157933 (+14 ms)Region opened successfully at 1732416157935 (+2 ms) 2024-11-24T02:42:37,937 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., pid=37, masterSystemTime=1732416157887 2024-11-24T02:42:37,939 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,939 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:37,944 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:37,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:42:37,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-24T02:42:37,950 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 in 213 msec 2024-11-24T02:42:37,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T02:42:37,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, REOPEN/MOVE in 564 msec 2024-11-24T02:42:37,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-24T02:42:37,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 577 msec 2024-11-24T02:42:37,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0210 sec 2024-11-24T02:42:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T02:42:37,966 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:37,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c00921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:37,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:37,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c70cee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:37,972 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:37,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc1b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b1860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:37,981 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:37,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6668979c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:37,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145a12b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c7c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:37,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:37,998 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:37,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e8c285, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:37,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,002 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a409486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52903b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9731fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbfabba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,028 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564ecc2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,031 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5d234c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,036 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d11f773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,039 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160ed270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c993de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,049 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6d8012, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf07c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,052 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e016d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,054 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10fe0f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,056 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fb429c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,056 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:42:38,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:42:38,058 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:42:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:42:38,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a03c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:42:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:38,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:42:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237a3912, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:42:38,065 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,066 DEBUG [Thread-688 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,067 DEBUG [Thread-688 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,068 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,069 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,069 DEBUG [Thread-695 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,070 DEBUG [Thread-695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,081 DEBUG [Thread-699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,083 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:38,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-24T02:42:38,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:38,094 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:38,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:38,097 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:38,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:42:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:38,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:38,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:38,116 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,116 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,118 DEBUG [Thread-693 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,137 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,181 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,181 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:42:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:42:38,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:42:38,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:42:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:38,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:42:38,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ebc9b1d4c3eb4f95b69a028e42325960_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416158106/Put/seqid=0 2024-11-24T02:42:38,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416158169, max timestamp=1732416158236Num uniques 859; 2024-11-24T02:42:38,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:38,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416158169, max timestamp=1732416158236Num uniques 859; 2024-11-24T02:42:38,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:38,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416158169, max timestamp=1732416158236Num uniques 859; 2024-11-24T02:42:38,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:42:38,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:38,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:38,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741963_1139 (size=14594) 2024-11-24T02:42:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416218317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416218324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416218326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416218330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416218332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:38,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:38,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:38,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416218439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416218439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416218449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416218449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416218450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416218649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416218649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416218660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416218660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416218660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,684 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:38,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ebc9b1d4c3eb4f95b69a028e42325960_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ebc9b1d4c3eb4f95b69a028e42325960_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:38,695 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fc937456e1e6454590b72345995c1588, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:38,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fc937456e1e6454590b72345995c1588 is 175, key is test_row_0/A:col10/1732416158106/Put/seqid=0 2024-11-24T02:42:38,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:38,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:38,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741964_1140 (size=39549) 2024-11-24T02:42:38,765 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fc937456e1e6454590b72345995c1588 2024-11-24T02:42:38,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c53467796bfe42048bcee4b9680c8fc3 is 50, key is test_row_0/B:col10/1732416158106/Put/seqid=0 2024-11-24T02:42:38,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:38,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:38,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741965_1141 (size=12001) 2024-11-24T02:42:38,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c53467796bfe42048bcee4b9680c8fc3 2024-11-24T02:42:38,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/6e3cc7c33ed84fd6979353ce47fb72ef is 50, key is test_row_0/C:col10/1732416158106/Put/seqid=0 2024-11-24T02:42:38,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416218959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416218959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741966_1142 (size=12001) 2024-11-24T02:42:38,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/6e3cc7c33ed84fd6979353ce47fb72ef 2024-11-24T02:42:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416218979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416218979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fc937456e1e6454590b72345995c1588 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588 2024-11-24T02:42:38,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416218979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:38,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588, entries=200, sequenceid=14, filesize=38.6 K 2024-11-24T02:42:38,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c53467796bfe42048bcee4b9680c8fc3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3 2024-11-24T02:42:39,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T02:42:39,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:39,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:39,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:39,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/6e3cc7c33ed84fd6979353ce47fb72ef as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef 2024-11-24T02:42:39,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T02:42:39,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a7378d33aaa14aba026a2abf2aaf9320 in 931ms, sequenceid=14, compaction requested=false 2024-11-24T02:42:39,040 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:42:39,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:39,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:42:39,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:39,179 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T02:42:39,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:39,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fc25a2630b8a456287c5d2ad3c88b8e7_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416158274/Put/seqid=0 2024-11-24T02:42:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:39,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741967_1143 (size=12154) 2024-11-24T02:42:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:39,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fc25a2630b8a456287c5d2ad3c88b8e7_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc25a2630b8a456287c5d2ad3c88b8e7_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:39,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1f08984d3b70458dbfed12e0d295d4d6, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1f08984d3b70458dbfed12e0d295d4d6 is 175, key is test_row_0/A:col10/1732416158274/Put/seqid=0 2024-11-24T02:42:39,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741968_1144 (size=30955) 2024-11-24T02:42:39,325 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1f08984d3b70458dbfed12e0d295d4d6 2024-11-24T02:42:39,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/673a746c6aca495397a5e9f370d28297 is 50, key is test_row_0/B:col10/1732416158274/Put/seqid=0 2024-11-24T02:42:39,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741969_1145 (size=12001) 2024-11-24T02:42:39,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/673a746c6aca495397a5e9f370d28297 2024-11-24T02:42:39,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/8ca49f5ff41242ff8e21e49e3171c46e is 50, key is test_row_0/C:col10/1732416158274/Put/seqid=0 2024-11-24T02:42:39,460 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:42:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:39,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:39,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741970_1146 (size=12001) 2024-11-24T02:42:39,489 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/8ca49f5ff41242ff8e21e49e3171c46e 2024-11-24T02:42:39,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1f08984d3b70458dbfed12e0d295d4d6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6 2024-11-24T02:42:39,506 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6, entries=150, sequenceid=49, filesize=30.2 K 2024-11-24T02:42:39,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/673a746c6aca495397a5e9f370d28297 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297 2024-11-24T02:42:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416219505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,519 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T02:42:39,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416219508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/8ca49f5ff41242ff8e21e49e3171c46e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e 2024-11-24T02:42:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416219510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,528 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T02:42:39,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416219512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,530 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for a7378d33aaa14aba026a2abf2aaf9320 in 351ms, sequenceid=49, compaction requested=false 2024-11-24T02:42:39,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:39,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:39,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-24T02:42:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-24T02:42:39,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:39,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:39,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:39,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:39,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:39,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:39,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:39,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T02:42:39,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4380 sec 2024-11-24T02:42:39,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4530 sec 2024-11-24T02:42:39,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112427cfcf09341d4b62af133da0a03ed441_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741971_1147 (size=14594) 2024-11-24T02:42:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416219650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416219654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416219656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416219662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416219662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416219769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416219769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416219769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416219779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416219779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416219989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416219989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416219990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416219990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416219999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:40,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112427cfcf09341d4b62af133da0a03ed441_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112427cfcf09341d4b62af133da0a03ed441_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:40,040 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f4f6b69ba5bb4ba380d2c394aa802eac, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:40,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f4f6b69ba5bb4ba380d2c394aa802eac is 175, key is test_row_0/A:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:40,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741972_1148 (size=39549) 2024-11-24T02:42:40,063 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f4f6b69ba5bb4ba380d2c394aa802eac 2024-11-24T02:42:40,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/4adce482827b436187d3e63315267f25 is 50, key is test_row_0/B:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:40,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741973_1149 (size=12001) 2024-11-24T02:42:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:42:40,250 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:40,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:40,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-24T02:42:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:40,256 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:40,257 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:40,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:40,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416220299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416220299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416220299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416220310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416220319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:40,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/4adce482827b436187d3e63315267f25 2024-11-24T02:42:40,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/436b49b61c9f45eb9c4f542dbbf829e9 is 50, key is test_row_0/C:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:40,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:40,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:40,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741974_1150 (size=12001) 2024-11-24T02:42:40,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416220819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416220819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416220829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416220830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416220839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:40,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:40,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:40,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:40,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/436b49b61c9f45eb9c4f542dbbf829e9 2024-11-24T02:42:40,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f4f6b69ba5bb4ba380d2c394aa802eac as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac 2024-11-24T02:42:41,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac, entries=200, sequenceid=61, filesize=38.6 K 2024-11-24T02:42:41,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/4adce482827b436187d3e63315267f25 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25 2024-11-24T02:42:41,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T02:42:41,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/436b49b61c9f45eb9c4f542dbbf829e9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9 2024-11-24T02:42:41,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:41,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:41,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:41,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T02:42:41,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a7378d33aaa14aba026a2abf2aaf9320 in 1515ms, sequenceid=61, compaction requested=true 2024-11-24T02:42:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:41,049 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:41,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:41,049 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:41,054 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:41,054 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:41,055 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,055 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.2 K 2024-11-24T02:42:41,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:41,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:41,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:41,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:41,057 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:41,057 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:41,057 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,057 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=107.5 K 2024-11-24T02:42:41,057 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,058 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac] 2024-11-24T02:42:41,058 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c53467796bfe42048bcee4b9680c8fc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416158094 2024-11-24T02:42:41,059 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc937456e1e6454590b72345995c1588, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416158094 2024-11-24T02:42:41,059 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 673a746c6aca495397a5e9f370d28297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416158274 2024-11-24T02:42:41,059 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f08984d3b70458dbfed12e0d295d4d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416158274 2024-11-24T02:42:41,060 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adce482827b436187d3e63315267f25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:41,061 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4f6b69ba5bb4ba380d2c394aa802eac, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:41,091 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:41,092 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/24f03333c4b24620bee87826250b31cd is 50, key is test_row_0/B:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:41,095 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:41,124 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d750e3ac9cc24624bbb6f0da6407278d_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741975_1151 (size=12104) 2024-11-24T02:42:41,158 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d750e3ac9cc24624bbb6f0da6407278d_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:41,158 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d750e3ac9cc24624bbb6f0da6407278d_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741976_1152 (size=4469) 2024-11-24T02:42:41,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:41,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112460d51249a16f4456ae23bb2ecf3516dc_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416159655/Put/seqid=0 2024-11-24T02:42:41,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741977_1153 (size=12154) 2024-11-24T02:42:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:41,290 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112460d51249a16f4456ae23bb2ecf3516dc_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112460d51249a16f4456ae23bb2ecf3516dc_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:41,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/64ce5545601c409ba8c8200001589b4c, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:41,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/64ce5545601c409ba8c8200001589b4c is 175, key is test_row_0/A:col10/1732416159655/Put/seqid=0 2024-11-24T02:42:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741978_1154 (size=30955) 2024-11-24T02:42:41,335 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/64ce5545601c409ba8c8200001589b4c 2024-11-24T02:42:41,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/db4592e8fbc543099ae98b96fe20d20e is 50, key is test_row_0/B:col10/1732416159655/Put/seqid=0 2024-11-24T02:42:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741979_1155 (size=12001) 2024-11-24T02:42:41,575 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/24f03333c4b24620bee87826250b31cd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24f03333c4b24620bee87826250b31cd 2024-11-24T02:42:41,585 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into 24f03333c4b24620bee87826250b31cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:41,585 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:41,585 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=13, startTime=1732416161049; duration=0sec 2024-11-24T02:42:41,585 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:41,585 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:41,586 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:41,590 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:41,590 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:41,590 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,591 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.2 K 2024-11-24T02:42:41,591 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3cc7c33ed84fd6979353ce47fb72ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416158094 2024-11-24T02:42:41,594 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#133 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:41,594 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ca49f5ff41242ff8e21e49e3171c46e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416158274 2024-11-24T02:42:41,595 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 436b49b61c9f45eb9c4f542dbbf829e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:41,595 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/b0a3e9f8719346019ee19eb033340f2b is 175, key is test_row_0/A:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:41,626 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:41,626 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/c8fde89bd2af44b4863403db1326a50b is 50, key is test_row_0/C:col10/1732416159528/Put/seqid=0 2024-11-24T02:42:41,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741980_1156 (size=31058) 2024-11-24T02:42:41,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741981_1157 (size=12104) 2024-11-24T02:42:41,673 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/c8fde89bd2af44b4863403db1326a50b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/c8fde89bd2af44b4863403db1326a50b 2024-11-24T02:42:41,679 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into c8fde89bd2af44b4863403db1326a50b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:41,679 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:41,679 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=13, startTime=1732416161055; duration=0sec 2024-11-24T02:42:41,679 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:41,679 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:41,799 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/db4592e8fbc543099ae98b96fe20d20e 2024-11-24T02:42:41,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/b32b1bc6f16348009daaf9c7600ddb5f is 50, key is test_row_0/C:col10/1732416159655/Put/seqid=0 2024-11-24T02:42:41,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741982_1158 (size=12001) 2024-11-24T02:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:41,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:41,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/b32b1bc6f16348009daaf9c7600ddb5f 2024-11-24T02:42:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/64ce5545601c409ba8c8200001589b4c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c 2024-11-24T02:42:41,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416221856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416221856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416221859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:41,879 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416221859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:41,882 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c, entries=150, sequenceid=81, filesize=30.2 K 2024-11-24T02:42:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/db4592e8fbc543099ae98b96fe20d20e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e 2024-11-24T02:42:41,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416221878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:41,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:41,890 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T02:42:41,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/b32b1bc6f16348009daaf9c7600ddb5f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f 2024-11-24T02:42:41,898 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T02:42:41,900 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a7378d33aaa14aba026a2abf2aaf9320 in 705ms, sequenceid=81, compaction requested=false 2024-11-24T02:42:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-24T02:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-24T02:42:41,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-24T02:42:41,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-24T02:42:41,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6560 sec 2024-11-24T02:42:41,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:42:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:41,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:42,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112458bfaafc30c64ab39fdc31f1b3d7a5e7_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416222032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416222036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,046 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/b0a3e9f8719346019ee19eb033340f2b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b 2024-11-24T02:42:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416222041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416222045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,055 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into b0a3e9f8719346019ee19eb033340f2b(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:42,055 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:42,055 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=13, startTime=1732416161049; duration=0sec 2024-11-24T02:42:42,056 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:42,056 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:42,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741983_1159 (size=17034) 2024-11-24T02:42:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416222149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416222149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416222159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416222159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416222359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416222359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416222369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416222371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:42:42,409 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:42,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:42,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T02:42:42,414 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:42,415 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:42,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:42,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:42,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112458bfaafc30c64ab39fdc31f1b3d7a5e7_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458bfaafc30c64ab39fdc31f1b3d7a5e7_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:42,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/d5908851893e4b9b94a02e6d5ae1cc55, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:42,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/d5908851893e4b9b94a02e6d5ae1cc55 is 175, key is test_row_0/A:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:42,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741984_1160 (size=48139) 2024-11-24T02:42:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:42,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:42,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:42,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416222669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416222669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416222679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416222679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:42,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:42,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:42,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/d5908851893e4b9b94a02e6d5ae1cc55 2024-11-24T02:42:42,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2b2382a46c794ab7b402b9003112ea34 is 50, key is test_row_0/B:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:42,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741985_1161 (size=12001) 2024-11-24T02:42:42,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2b2382a46c794ab7b402b9003112ea34 2024-11-24T02:42:42,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/31a45940b2bd43609ad3dbf654d0e62c is 50, key is test_row_0/C:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:42,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:42,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:42,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741986_1162 (size=12001) 2024-11-24T02:42:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:43,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416223179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:43,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416223179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416223189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:43,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416223189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:43,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:43,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/31a45940b2bd43609ad3dbf654d0e62c 2024-11-24T02:42:43,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/d5908851893e4b9b94a02e6d5ae1cc55 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55 2024-11-24T02:42:43,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55, entries=250, sequenceid=97, filesize=47.0 K 2024-11-24T02:42:43,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2b2382a46c794ab7b402b9003112ea34 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34 2024-11-24T02:42:43,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T02:42:43,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/31a45940b2bd43609ad3dbf654d0e62c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c 2024-11-24T02:42:43,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T02:42:43,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7378d33aaa14aba026a2abf2aaf9320 in 1376ms, sequenceid=97, compaction requested=true 2024-11-24T02:42:43,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:43,370 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:43,370 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:43,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:43,371 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:43,371 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:43,371 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:43,371 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:43,371 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,371 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,371 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=107.6 K 2024-11-24T02:42:43,371 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24f03333c4b24620bee87826250b31cd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.3 K 2024-11-24T02:42:43,371 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,371 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55] 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f03333c4b24620bee87826250b31cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a3e9f8719346019ee19eb033340f2b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting db4592e8fbc543099ae98b96fe20d20e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732416159653 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ce5545601c409ba8c8200001589b4c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732416159653 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2382a46c794ab7b402b9003112ea34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:43,372 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5908851893e4b9b94a02e6d5ae1cc55, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:43,381 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:43,383 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:43,383 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/257c968aa9c84c13ae9d373a8d095cca is 50, key is test_row_0/B:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:43,397 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240a2a75071e704dd2816f7af43872c164_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:43,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:42:43,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,399 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:42:43,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:43,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:43,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741987_1163 (size=12207) 2024-11-24T02:42:43,404 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240a2a75071e704dd2816f7af43872c164_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:43,404 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240a2a75071e704dd2816f7af43872c164_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:43,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124335fd87a815e4e099f0f2103c17d2b01_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416162021/Put/seqid=0 2024-11-24T02:42:43,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741988_1164 (size=4469) 2024-11-24T02:42:43,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741989_1165 (size=12154) 2024-11-24T02:42:43,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:43,811 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/257c968aa9c84c13ae9d373a8d095cca as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/257c968aa9c84c13ae9d373a8d095cca 2024-11-24T02:42:43,820 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into 257c968aa9c84c13ae9d373a8d095cca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:43,821 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:43,821 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=13, startTime=1732416163370; duration=0sec 2024-11-24T02:42:43,821 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:43,821 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:43,821 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:43,822 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:43,829 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:43,829 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:43,829 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/c8fde89bd2af44b4863403db1326a50b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.3 K 2024-11-24T02:42:43,831 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c8fde89bd2af44b4863403db1326a50b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416159502 2024-11-24T02:42:43,832 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#142 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:43,832 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/cd7c7f8b3e814fca8d63c1a60901e96d is 175, key is test_row_0/A:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:43,835 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b32b1bc6f16348009daaf9c7600ddb5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732416159653 2024-11-24T02:42:43,836 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a45940b2bd43609ad3dbf654d0e62c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:43,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:43,849 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124335fd87a815e4e099f0f2103c17d2b01_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124335fd87a815e4e099f0f2103c17d2b01_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fff2487302a441349dc6a3a1365dd40c, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:43,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fff2487302a441349dc6a3a1365dd40c is 175, key is test_row_0/A:col10/1732416162021/Put/seqid=0 2024-11-24T02:42:43,860 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:43,861 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/72f66cfff09e4d0280911d6786899615 is 50, key is test_row_0/C:col10/1732416161992/Put/seqid=0 2024-11-24T02:42:43,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741991_1167 (size=30955) 2024-11-24T02:42:43,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741990_1166 (size=31161) 2024-11-24T02:42:43,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:43,922 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/cd7c7f8b3e814fca8d63c1a60901e96d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d 2024-11-24T02:42:43,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741992_1168 (size=12207) 2024-11-24T02:42:43,931 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/72f66cfff09e4d0280911d6786899615 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/72f66cfff09e4d0280911d6786899615 2024-11-24T02:42:43,935 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into cd7c7f8b3e814fca8d63c1a60901e96d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:43,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:43,935 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=13, startTime=1732416163369; duration=0sec 2024-11-24T02:42:43,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:43,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:43,940 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into 72f66cfff09e4d0280911d6786899615(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:43,940 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:43,940 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=13, startTime=1732416163370; duration=0sec 2024-11-24T02:42:43,940 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:43,940 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416223960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:43,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416224069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416224199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416224199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416224199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416224199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,200 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:44,200 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,200 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,200 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416224279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fff2487302a441349dc6a3a1365dd40c 2024-11-24T02:42:44,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2f5d7b4f239b43e99098ff8b350f3ec7 is 50, key is test_row_0/B:col10/1732416162021/Put/seqid=0 2024-11-24T02:42:44,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741993_1169 (size=12001) 2024-11-24T02:42:44,314 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2f5d7b4f239b43e99098ff8b350f3ec7 2024-11-24T02:42:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/bef6d98ec16645a08ea48394f9af9240 is 50, key is test_row_0/C:col10/1732416162021/Put/seqid=0 2024-11-24T02:42:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741994_1170 (size=12001) 2024-11-24T02:42:44,339 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/bef6d98ec16645a08ea48394f9af9240 2024-11-24T02:42:44,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/fff2487302a441349dc6a3a1365dd40c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c 2024-11-24T02:42:44,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c, entries=150, sequenceid=116, filesize=30.2 K 2024-11-24T02:42:44,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2f5d7b4f239b43e99098ff8b350f3ec7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7 2024-11-24T02:42:44,373 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:42:44,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/bef6d98ec16645a08ea48394f9af9240 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240 2024-11-24T02:42:44,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:42:44,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a7378d33aaa14aba026a2abf2aaf9320 in 985ms, sequenceid=116, compaction requested=false 2024-11-24T02:42:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:44,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T02:42:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T02:42:44,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T02:42:44,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9710 sec 2024-11-24T02:42:44,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9800 sec 2024-11-24T02:42:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:42:44,559 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:44,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T02:42:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:44,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:44,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:44,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:44,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:44,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124de92016413814986be3521b223a14d53_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:44,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741995_1171 (size=14744) 2024-11-24T02:42:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416224644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:44,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:44,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416224759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:44,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:44,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:44,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:44,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416224969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:45,010 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:45,015 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124de92016413814986be3521b223a14d53_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de92016413814986be3521b223a14d53_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:45,016 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f0f19d6869b6448eb02abc580101d4ec, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:45,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f0f19d6869b6448eb02abc580101d4ec is 175, key is test_row_0/A:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741996_1172 (size=39699) 2024-11-24T02:42:45,028 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f0f19d6869b6448eb02abc580101d4ec 2024-11-24T02:42:45,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/362350ccc6ef49ef80f71a7242b39586 is 50, key is test_row_0/B:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741997_1173 (size=12101) 2024-11-24T02:42:45,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/362350ccc6ef49ef80f71a7242b39586 2024-11-24T02:42:45,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/10a8c50017524f4d9525a3b6a2de5d3c is 50, key is test_row_0/C:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741998_1174 (size=12101) 2024-11-24T02:42:45,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:45,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:45,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:45,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416225278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:45,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:45,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:45,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/10a8c50017524f4d9525a3b6a2de5d3c 2024-11-24T02:42:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:45,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/f0f19d6869b6448eb02abc580101d4ec as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec 2024-11-24T02:42:45,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec, entries=200, sequenceid=134, filesize=38.8 K 2024-11-24T02:42:45,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/362350ccc6ef49ef80f71a7242b39586 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586 2024-11-24T02:42:45,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T02:42:45,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/10a8c50017524f4d9525a3b6a2de5d3c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c 2024-11-24T02:42:45,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c, entries=150, sequenceid=134, filesize=11.8 K 2024-11-24T02:42:45,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a7378d33aaa14aba026a2abf2aaf9320 in 933ms, sequenceid=134, compaction requested=true 2024-11-24T02:42:45,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:45,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:45,524 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:45,524 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:45,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:45,533 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:45,533 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:45,533 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:45,533 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:45,533 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,533 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,533 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=99.4 K 2024-11-24T02:42:45,533 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,533 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec] 2024-11-24T02:42:45,533 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/257c968aa9c84c13ae9d373a8d095cca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.5 K 2024-11-24T02:42:45,534 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd7c7f8b3e814fca8d63c1a60901e96d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:45,534 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 257c968aa9c84c13ae9d373a8d095cca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:45,535 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5d7b4f239b43e99098ff8b350f3ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416162021 2024-11-24T02:42:45,535 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff2487302a441349dc6a3a1365dd40c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416162021 2024-11-24T02:42:45,536 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f19d6869b6448eb02abc580101d4ec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:45,536 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 362350ccc6ef49ef80f71a7242b39586, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:45,560 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:45,561 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:45,561 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/24d3a78af3c643c898891a0f123c9d5d is 50, key is test_row_0/B:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,569 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124246ceca2d38940f8b374ab330343f301_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:45,572 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124246ceca2d38940f8b374ab330343f301_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:45,572 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124246ceca2d38940f8b374ab330343f301_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:45,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741999_1175 (size=12409) 2024-11-24T02:42:45,589 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/24d3a78af3c643c898891a0f123c9d5d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24d3a78af3c643c898891a0f123c9d5d 2024-11-24T02:42:45,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742000_1176 (size=4469) 2024-11-24T02:42:45,598 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#151 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:45,599 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1d6417d336884683a0d57991339becb8 is 175, key is test_row_0/A:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,602 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into 24d3a78af3c643c898891a0f123c9d5d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:45,602 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:45,602 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=13, startTime=1732416165524; duration=0sec 2024-11-24T02:42:45,603 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:45,603 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:45,603 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:45,604 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:45,604 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:45,604 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,605 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/72f66cfff09e4d0280911d6786899615, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=35.5 K 2024-11-24T02:42:45,605 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 72f66cfff09e4d0280911d6786899615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416161853 2024-11-24T02:42:45,606 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting bef6d98ec16645a08ea48394f9af9240, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416162021 2024-11-24T02:42:45,606 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a8c50017524f4d9525a3b6a2de5d3c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:45,617 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:45,617 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/5395557d7911421f86c1a144c97b7a03 is 50, key is test_row_0/C:col10/1732416164589/Put/seqid=0 2024-11-24T02:42:45,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742001_1177 (size=31363) 2024-11-24T02:42:45,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742002_1178 (size=12409) 2024-11-24T02:42:45,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:42:45,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:45,649 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:45,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:45,658 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/5395557d7911421f86c1a144c97b7a03 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/5395557d7911421f86c1a144c97b7a03 2024-11-24T02:42:45,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bb7cebddfc414666a659fe4ca7aeb5d6_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416164643/Put/seqid=0 2024-11-24T02:42:45,669 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into 5395557d7911421f86c1a144c97b7a03(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:45,669 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:45,669 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=13, startTime=1732416165524; duration=0sec 2024-11-24T02:42:45,669 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:45,669 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:45,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742003_1179 (size=12304) 2024-11-24T02:42:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:45,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416225840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:45,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416225949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,026 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1d6417d336884683a0d57991339becb8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8 2024-11-24T02:42:46,032 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into 1d6417d336884683a0d57991339becb8(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:46,032 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:46,032 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=13, startTime=1732416165523; duration=0sec 2024-11-24T02:42:46,032 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:46,032 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:46,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:46,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bb7cebddfc414666a659fe4ca7aeb5d6_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb7cebddfc414666a659fe4ca7aeb5d6_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1c51fbdca99f4c67bca3d5a24e4fc3f0, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 is 175, key is test_row_0/A:col10/1732416164643/Put/seqid=0 2024-11-24T02:42:46,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742004_1180 (size=31105) 2024-11-24T02:42:46,097 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 2024-11-24T02:42:46,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/a134886ed878425696c47959e02837a8 is 50, key is test_row_0/B:col10/1732416164643/Put/seqid=0 2024-11-24T02:42:46,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742005_1181 (size=12151) 2024-11-24T02:42:46,116 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/a134886ed878425696c47959e02837a8 2024-11-24T02:42:46,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/f07ba1e6245147c5b0a5e413ec93e9b8 is 50, key is test_row_0/C:col10/1732416164643/Put/seqid=0 2024-11-24T02:42:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742006_1182 (size=12151) 2024-11-24T02:42:46,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416226159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416226218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416226219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,223 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416226219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,224 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,224 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416226229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,231 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416226469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:46,529 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/f07ba1e6245147c5b0a5e413ec93e9b8 2024-11-24T02:42:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 2024-11-24T02:42:46,538 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0, entries=150, sequenceid=152, filesize=30.4 K 2024-11-24T02:42:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/a134886ed878425696c47959e02837a8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8 2024-11-24T02:42:46,544 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T02:42:46,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/f07ba1e6245147c5b0a5e413ec93e9b8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8 2024-11-24T02:42:46,549 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T02:42:46,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a7378d33aaa14aba026a2abf2aaf9320 in 901ms, sequenceid=152, compaction requested=false 2024-11-24T02:42:46,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:46,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T02:42:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T02:42:46,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T02:42:46,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9900 sec 2024-11-24T02:42:46,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9940 sec 2024-11-24T02:42:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:42:46,709 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:46,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T02:42:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:46,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:46,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:46,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:46,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:42:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:46,864 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:46,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:46,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fcda7cb7b9ed4f8c9128ebc7403c31f4_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416165839/Put/seqid=0 2024-11-24T02:42:46,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742007_1183 (size=12304) 2024-11-24T02:42:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:47,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416227030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:47,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416227139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:47,284 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fcda7cb7b9ed4f8c9128ebc7403c31f4_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fcda7cb7b9ed4f8c9128ebc7403c31f4_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c8a09aedc4084161a8f71a55f7a0e91d, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:47,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c8a09aedc4084161a8f71a55f7a0e91d is 175, key is test_row_0/A:col10/1732416165839/Put/seqid=0 2024-11-24T02:42:47,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742008_1184 (size=31105) 2024-11-24T02:42:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416227349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:47,371 INFO [master/7c69a60bd8f6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T02:42:47,371 INFO [master/7c69a60bd8f6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T02:42:47,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416227659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:47,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:47,700 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c8a09aedc4084161a8f71a55f7a0e91d 2024-11-24T02:42:47,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/ebc19c304bd1441ea3a88d685b81b8ac is 50, key is test_row_0/B:col10/1732416165839/Put/seqid=0 2024-11-24T02:42:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742009_1185 (size=12151) 2024-11-24T02:42:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:48,117 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/ebc19c304bd1441ea3a88d685b81b8ac 2024-11-24T02:42:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/7e1863ce281246ebab29f615ca2712c4 is 50, key is test_row_0/C:col10/1732416165839/Put/seqid=0 2024-11-24T02:42:48,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742010_1186 (size=12151) 2024-11-24T02:42:48,128 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/7e1863ce281246ebab29f615ca2712c4 2024-11-24T02:42:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c8a09aedc4084161a8f71a55f7a0e91d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d 2024-11-24T02:42:48,137 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d, entries=150, sequenceid=169, filesize=30.4 K 2024-11-24T02:42:48,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/ebc19c304bd1441ea3a88d685b81b8ac as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac 2024-11-24T02:42:48,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T02:42:48,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/7e1863ce281246ebab29f615ca2712c4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4 2024-11-24T02:42:48,150 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T02:42:48,151 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a7378d33aaa14aba026a2abf2aaf9320 in 1287ms, sequenceid=169, compaction requested=true 2024-11-24T02:42:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T02:42:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T02:42:48,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T02:42:48,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-24T02:42:48,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4450 sec 2024-11-24T02:42:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:48,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:48,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:48,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5f6b79b122d40dfb4eb99bd5033924d_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:48,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742011_1187 (size=14794) 2024-11-24T02:42:48,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416228228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416228339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416228549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:48,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:48,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5f6b79b122d40dfb4eb99bd5033924d_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f6b79b122d40dfb4eb99bd5033924d_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:48,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c4536d1bbe0541ff984063ebe35d3795, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:48,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c4536d1bbe0541ff984063ebe35d3795 is 175, key is test_row_0/A:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:48,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742012_1188 (size=39749) 2024-11-24T02:42:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:42:48,859 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:48,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:48,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:48,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416228859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:48,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T02:42:48,861 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:48,862 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:48,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:48,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c4536d1bbe0541ff984063ebe35d3795 2024-11-24T02:42:49,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/22744c1c970e4c35b42d83a9c46af5a9 is 50, key is test_row_0/B:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742013_1189 (size=12151) 2024-11-24T02:42:49,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:49,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416229369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:49,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:49,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:49,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:49,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/22744c1c970e4c35b42d83a9c46af5a9 2024-11-24T02:42:49,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/735c11e0e7eb45beafaea9bd3fbd4330 is 50, key is test_row_0/C:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:49,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742014_1190 (size=12151) 2024-11-24T02:42:49,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:49,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:49,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/735c11e0e7eb45beafaea9bd3fbd4330 2024-11-24T02:42:49,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/c4536d1bbe0541ff984063ebe35d3795 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795 2024-11-24T02:42:49,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795, entries=200, sequenceid=187, filesize=38.8 K 2024-11-24T02:42:49,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/22744c1c970e4c35b42d83a9c46af5a9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9 2024-11-24T02:42:49,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T02:42:49,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/735c11e0e7eb45beafaea9bd3fbd4330 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330 2024-11-24T02:42:49,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T02:42:49,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a7378d33aaa14aba026a2abf2aaf9320 in 1676ms, sequenceid=187, compaction requested=true 2024-11-24T02:42:49,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:49,849 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:49,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:49,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:49,849 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:49,851 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:49,851 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:49,851 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,851 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133322 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:49,851 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:49,851 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24d3a78af3c643c898891a0f123c9d5d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=47.7 K 2024-11-24T02:42:49,851 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,851 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=130.2 K 2024-11-24T02:42:49,851 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,851 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795] 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d3a78af3c643c898891a0f123c9d5d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6417d336884683a0d57991339becb8, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a134886ed878425696c47959e02837a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732416164628 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c51fbdca99f4c67bca3d5a24e4fc3f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732416164628 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ebc19c304bd1441ea3a88d685b81b8ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416165816 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8a09aedc4084161a8f71a55f7a0e91d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416165816 2024-11-24T02:42:49,852 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 22744c1c970e4c35b42d83a9c46af5a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:49,853 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4536d1bbe0541ff984063ebe35d3795, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:49,862 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:49,863 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:49,863 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/db1a1b393db148ff8293ace8d575f4e9 is 50, key is test_row_0/B:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:49,865 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240f2390df6fee4989be42c6b68e263811_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:49,867 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240f2390df6fee4989be42c6b68e263811_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:49,868 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240f2390df6fee4989be42c6b68e263811_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:49,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742015_1191 (size=12595) 2024-11-24T02:42:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742016_1192 (size=4469) 2024-11-24T02:42:49,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:42:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:49,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:49,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:49,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240071528dd6374a779ae0073c37ef811b_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416168218/Put/seqid=0 2024-11-24T02:42:49,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742017_1193 (size=12304) 2024-11-24T02:42:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:50,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:50,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416230270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416230271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416230271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416230272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,284 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/db1a1b393db148ff8293ace8d575f4e9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db1a1b393db148ff8293ace8d575f4e9 2024-11-24T02:42:50,289 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#163 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:50,289 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into db1a1b393db148ff8293ace8d575f4e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:50,289 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:50,289 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=12, startTime=1732416169849; duration=0sec 2024-11-24T02:42:50,289 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/6580296930ca4611a8438693df702fea is 175, key is test_row_0/A:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:50,289 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:50,289 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:50,289 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:50,292 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:50,292 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:50,292 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:50,293 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/5395557d7911421f86c1a144c97b7a03, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=47.7 K 2024-11-24T02:42:50,293 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5395557d7911421f86c1a144c97b7a03, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732416163942 2024-11-24T02:42:50,294 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f07ba1e6245147c5b0a5e413ec93e9b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732416164628 2024-11-24T02:42:50,294 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1863ce281246ebab29f615ca2712c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416165816 2024-11-24T02:42:50,294 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 735c11e0e7eb45beafaea9bd3fbd4330, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:50,304 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:50,305 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/fca492b68ab2450c85dd440ae0a855b4 is 50, key is test_row_0/C:col10/1732416167021/Put/seqid=0 2024-11-24T02:42:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742018_1194 (size=31549) 2024-11-24T02:42:50,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742019_1195 (size=12595) 2024-11-24T02:42:50,318 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/6580296930ca4611a8438693df702fea as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea 2024-11-24T02:42:50,323 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/fca492b68ab2450c85dd440ae0a855b4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fca492b68ab2450c85dd440ae0a855b4 2024-11-24T02:42:50,325 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into 6580296930ca4611a8438693df702fea(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:50,326 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:50,326 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=12, startTime=1732416169849; duration=0sec 2024-11-24T02:42:50,326 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:50,326 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:50,328 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into fca492b68ab2450c85dd440ae0a855b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:50,329 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:50,329 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=12, startTime=1732416169850; duration=0sec 2024-11-24T02:42:50,329 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:50,329 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:50,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:50,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240071528dd6374a779ae0073c37ef811b_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240071528dd6374a779ae0073c37ef811b_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:50,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/014144b6c04242ef89e20a022e0eb54e, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:50,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/014144b6c04242ef89e20a022e0eb54e is 175, key is test_row_0/A:col10/1732416168218/Put/seqid=0 2024-11-24T02:42:50,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742020_1196 (size=31105) 2024-11-24T02:42:50,367 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/014144b6c04242ef89e20a022e0eb54e 2024-11-24T02:42:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/679c755f0dc94195aae41d95cd68ff0c is 50, key is test_row_0/B:col10/1732416168218/Put/seqid=0 2024-11-24T02:42:50,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742021_1197 (size=12151) 2024-11-24T02:42:50,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416230378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416230378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416230379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,381 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416230379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416230379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416230589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416230590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416230590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416230590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,781 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/679c755f0dc94195aae41d95cd68ff0c 2024-11-24T02:42:50,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d3125cff2d174f378e76347ba8265f0c is 50, key is test_row_0/C:col10/1732416168218/Put/seqid=0 2024-11-24T02:42:50,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742022_1198 (size=12151) 2024-11-24T02:42:50,795 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d3125cff2d174f378e76347ba8265f0c 2024-11-24T02:42:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/014144b6c04242ef89e20a022e0eb54e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e 2024-11-24T02:42:50,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e, entries=150, sequenceid=201, filesize=30.4 K 2024-11-24T02:42:50,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/679c755f0dc94195aae41d95cd68ff0c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c 2024-11-24T02:42:50,822 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T02:42:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d3125cff2d174f378e76347ba8265f0c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c 2024-11-24T02:42:50,830 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T02:42:50,832 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a7378d33aaa14aba026a2abf2aaf9320 in 899ms, sequenceid=201, compaction requested=false 2024-11-24T02:42:50,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:50,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:50,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T02:42:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T02:42:50,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T02:42:50,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9720 sec 2024-11-24T02:42:50,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9780 sec 2024-11-24T02:42:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:50,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:42:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416170263, max timestamp=1732416170912Num uniques 859; 2024-11-24T02:42:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:50,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416170263, max timestamp=1732416170912Num uniques 859; 2024-11-24T02:42:50,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:50,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112406941c6ccc10416d9786d59cb738434e_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:50,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742023_1199 (size=14794) 2024-11-24T02:42:50,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416230943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416170940, max timestamp=1732416170944Num uniques 859; 2024-11-24T02:42:50,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:50,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416170940, max timestamp=1732416170944Num uniques 859; 2024-11-24T02:42:50,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:50,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416170940, max timestamp=1732416170944Num uniques 859; 2024-11-24T02:42:50,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:50,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416230981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416230983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:50,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416230984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:50,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:42:51,010 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:51,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T02:42:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:51,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:51,013 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:51,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:51,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416231049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416231099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416231099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416231109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:51,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416231258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416231309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416231309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416231328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:51,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112406941c6ccc10416d9786d59cb738434e_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112406941c6ccc10416d9786d59cb738434e_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:51,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3975e8fc399d47c19f17b65a08356116, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:51,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3975e8fc399d47c19f17b65a08356116 is 175, key is test_row_0/A:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:51,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742024_1200 (size=39749) 2024-11-24T02:42:51,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416231569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416231619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416231619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416231639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:51,746 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3975e8fc399d47c19f17b65a08356116 2024-11-24T02:42:51,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/13330cf633b04d59a4be0f31b6212f6f is 50, key is test_row_0/B:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:51,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742025_1201 (size=12151) 2024-11-24T02:42:51,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/13330cf633b04d59a4be0f31b6212f6f 2024-11-24T02:42:51,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/017a6420e4174c339948173680979833 is 50, key is test_row_0/C:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:51,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742026_1202 (size=12151) 2024-11-24T02:42:51,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:51,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:52,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:52,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416232079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416232129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416232129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:52,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416232149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/017a6420e4174c339948173680979833 2024-11-24T02:42:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3975e8fc399d47c19f17b65a08356116 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116 2024-11-24T02:42:52,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116, entries=200, sequenceid=225, filesize=38.8 K 2024-11-24T02:42:52,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/13330cf633b04d59a4be0f31b6212f6f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f 2024-11-24T02:42:52,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T02:42:52,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/017a6420e4174c339948173680979833 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833 2024-11-24T02:42:52,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T02:42:52,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for a7378d33aaa14aba026a2abf2aaf9320 in 1291ms, sequenceid=225, compaction requested=true 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:52,204 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:52,204 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:52,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:52,205 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:52,205 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:52,205 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:52,205 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:52,205 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,205 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=100.0 K 2024-11-24T02:42:52,205 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,205 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116] 2024-11-24T02:42:52,205 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,206 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db1a1b393db148ff8293ace8d575f4e9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=36.0 K 2024-11-24T02:42:52,206 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6580296930ca4611a8438693df702fea, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:52,206 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting db1a1b393db148ff8293ace8d575f4e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:52,206 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 679c755f0dc94195aae41d95cd68ff0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416168214 2024-11-24T02:42:52,206 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014144b6c04242ef89e20a022e0eb54e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416168214 2024-11-24T02:42:52,207 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3975e8fc399d47c19f17b65a08356116, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:52,207 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 13330cf633b04d59a4be0f31b6212f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:52,213 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:52,213 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:52,214 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/6ae5f9981a284971b11f234154f09bab is 50, key is test_row_0/B:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:52,224 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d0796c90040f41c0a2c42096e1302815_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:52,226 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d0796c90040f41c0a2c42096e1302815_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:52,226 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d0796c90040f41c0a2c42096e1302815_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:52,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742027_1203 (size=12697) 2024-11-24T02:42:52,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742028_1204 (size=4469) 2024-11-24T02:42:52,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:42:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T02:42:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:52,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112437d4a0f87e0640d4868c2176e54cfb02_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416170981/Put/seqid=0 2024-11-24T02:42:52,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742029_1205 (size=12454) 2024-11-24T02:42:52,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:52,260 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112437d4a0f87e0640d4868c2176e54cfb02_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437d4a0f87e0640d4868c2176e54cfb02_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/639abcc7d2da416fa0569c62d745dca6, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/639abcc7d2da416fa0569c62d745dca6 is 175, key is test_row_0/A:col10/1732416170981/Put/seqid=0 2024-11-24T02:42:52,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742030_1206 (size=31255) 2024-11-24T02:42:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:52,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:52,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416232434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416232549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:52,636 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/6ae5f9981a284971b11f234154f09bab as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/6ae5f9981a284971b11f234154f09bab 2024-11-24T02:42:52,636 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#172 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:52,637 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3f5ff48529294856b1bb0b35f6caf3e6 is 175, key is test_row_0/A:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:52,646 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into 6ae5f9981a284971b11f234154f09bab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:52,646 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:52,646 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=13, startTime=1732416172204; duration=0sec 2024-11-24T02:42:52,646 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:52,646 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:52,646 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:52,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742031_1207 (size=31651) 2024-11-24T02:42:52,647 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:52,647 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:52,648 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:52,648 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fca492b68ab2450c85dd440ae0a855b4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=36.0 K 2024-11-24T02:42:52,648 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fca492b68ab2450c85dd440ae0a855b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416167021 2024-11-24T02:42:52,649 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d3125cff2d174f378e76347ba8265f0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416168214 2024-11-24T02:42:52,649 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 017a6420e4174c339948173680979833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:52,652 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/3f5ff48529294856b1bb0b35f6caf3e6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6 2024-11-24T02:42:52,657 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:52,657 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into 3f5ff48529294856b1bb0b35f6caf3e6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:52,657 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:52,658 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=13, startTime=1732416172204; duration=0sec 2024-11-24T02:42:52,658 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/cf8575d7d6c4457e8049afbc61dc6062 is 50, key is test_row_0/C:col10/1732416170912/Put/seqid=0 2024-11-24T02:42:52,658 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:52,658 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:52,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742032_1208 (size=12697) 2024-11-24T02:42:52,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/639abcc7d2da416fa0569c62d745dca6 2024-11-24T02:42:52,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/7a35e0a2c5404b7ca9eb66446508b2dc is 50, key is test_row_0/B:col10/1732416170981/Put/seqid=0 2024-11-24T02:42:52,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742033_1209 (size=12301) 2024-11-24T02:42:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416232758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,072 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/cf8575d7d6c4457e8049afbc61dc6062 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/cf8575d7d6c4457e8049afbc61dc6062 2024-11-24T02:42:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416233069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,077 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into cf8575d7d6c4457e8049afbc61dc6062(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:53,077 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:53,077 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=13, startTime=1732416172204; duration=0sec 2024-11-24T02:42:53,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:53,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:53,093 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/7a35e0a2c5404b7ca9eb66446508b2dc 2024-11-24T02:42:53,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/0f3fa869672e4e49a66e42d7344eb89e is 50, key is test_row_0/C:col10/1732416170981/Put/seqid=0 2024-11-24T02:42:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416233099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,103 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742034_1210 (size=12301) 2024-11-24T02:42:53,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416233139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,140 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416233149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,153 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:53,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416233168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,173 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,515 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/0f3fa869672e4e49a66e42d7344eb89e 2024-11-24T02:42:53,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/639abcc7d2da416fa0569c62d745dca6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6 2024-11-24T02:42:53,525 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6, entries=150, sequenceid=260, filesize=30.5 K 2024-11-24T02:42:53,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/7a35e0a2c5404b7ca9eb66446508b2dc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc 2024-11-24T02:42:53,530 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc, entries=150, sequenceid=260, filesize=12.0 K 2024-11-24T02:42:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/0f3fa869672e4e49a66e42d7344eb89e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e 2024-11-24T02:42:53,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e, entries=150, sequenceid=260, filesize=12.0 K 2024-11-24T02:42:53,544 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a7378d33aaa14aba026a2abf2aaf9320 in 1308ms, sequenceid=260, compaction requested=false 2024-11-24T02:42:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:53,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T02:42:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T02:42:53,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T02:42:53,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5330 sec 2024-11-24T02:42:53,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.5370 sec 2024-11-24T02:42:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:53,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:42:53,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:53,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9b2381afb0c424695630632e185df84_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:53,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742035_1211 (size=14994) 2024-11-24T02:42:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416233661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416233769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:53,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416233978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:54,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:54,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9b2381afb0c424695630632e185df84_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9b2381afb0c424695630632e185df84_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:54,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/8232babd89c64272bea70df7ec9a0eae, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:54,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/8232babd89c64272bea70df7ec9a0eae is 175, key is test_row_0/A:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:54,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742036_1212 (size=39949) 2024-11-24T02:42:54,013 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/8232babd89c64272bea70df7ec9a0eae 2024-11-24T02:42:54,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/3c80b4fb3dd24afd961148a75ff572ae is 50, key is test_row_0/B:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:54,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742037_1213 (size=12301) 2024-11-24T02:42:54,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416234289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:54,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:54,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:54,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:54,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/3c80b4fb3dd24afd961148a75ff572ae 2024-11-24T02:42:54,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/acb0dc91d492422fa19713fa47601792 is 50, key is test_row_0/C:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:54,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742038_1214 (size=12301) 2024-11-24T02:42:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416234799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:54,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/acb0dc91d492422fa19713fa47601792 2024-11-24T02:42:54,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/8232babd89c64272bea70df7ec9a0eae as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae 2024-11-24T02:42:54,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae, entries=200, sequenceid=275, filesize=39.0 K 2024-11-24T02:42:54,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/3c80b4fb3dd24afd961148a75ff572ae as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae 2024-11-24T02:42:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T02:42:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/acb0dc91d492422fa19713fa47601792 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792 2024-11-24T02:42:54,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T02:42:54,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a7378d33aaa14aba026a2abf2aaf9320 in 1283ms, sequenceid=275, compaction requested=true 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:54,863 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:54,863 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:54,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:54,864 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:54,864 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:54,864 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:54,864 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:54,864 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:54,864 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:54,865 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=100.4 K 2024-11-24T02:42:54,865 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/6ae5f9981a284971b11f234154f09bab, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=36.4 K 2024-11-24T02:42:54,865 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae] 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5ff48529294856b1bb0b35f6caf3e6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae5f9981a284971b11f234154f09bab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a35e0a2c5404b7ca9eb66446508b2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416170980 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 639abcc7d2da416fa0569c62d745dca6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416170980 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c80b4fb3dd24afd961148a75ff572ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:54,865 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8232babd89c64272bea70df7ec9a0eae, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:54,871 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:54,880 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240dfa16328a7a46b78d514730e9d46a88_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:54,881 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240dfa16328a7a46b78d514730e9d46a88_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:54,881 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240dfa16328a7a46b78d514730e9d46a88_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:54,881 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:54,882 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/94b76ef708a64c9c927a7d4a8fb0935c is 50, key is test_row_0/B:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:54,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742039_1215 (size=4469) 2024-11-24T02:42:54,890 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#180 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:54,890 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/79955c4d15bf4e6aa82f8e5e9bcf2543 is 175, key is test_row_0/A:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:54,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742040_1216 (size=12949) 2024-11-24T02:42:54,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742041_1217 (size=31903) 2024-11-24T02:42:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:55,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:55,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:55,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a47bd5c3e9ac42598e0f1a2d33172fef_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416175121/Put/seqid=0 2024-11-24T02:42:55,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742042_1218 (size=12454) 2024-11-24T02:42:55,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:55,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a47bd5c3e9ac42598e0f1a2d33172fef_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a47bd5c3e9ac42598e0f1a2d33172fef_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:55,150 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/13cc48d0e3a94ed68570171e719561f1, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:55,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/13cc48d0e3a94ed68570171e719561f1 is 175, key is test_row_0/A:col10/1732416175121/Put/seqid=0 2024-11-24T02:42:55,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742043_1219 (size=31255) 2024-11-24T02:42:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416235155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416235155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/13cc48d0e3a94ed68570171e719561f1 2024-11-24T02:42:55,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/79e1603bf1d8400a850fc9199acd2778 is 50, key is test_row_0/B:col10/1732416175121/Put/seqid=0 2024-11-24T02:42:55,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742044_1220 (size=12301) 2024-11-24T02:42:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:42:55,179 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:55,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:42:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-24T02:42:55,181 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:42:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:55,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:42:55,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:42:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416235179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,183 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416235189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,197 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416235269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416235269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:55,299 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/94b76ef708a64c9c927a7d4a8fb0935c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/94b76ef708a64c9c927a7d4a8fb0935c 2024-11-24T02:42:55,306 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into 94b76ef708a64c9c927a7d4a8fb0935c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:55,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:55,306 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=13, startTime=1732416174863; duration=0sec 2024-11-24T02:42:55,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:55,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:55,306 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:42:55,308 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/79955c4d15bf4e6aa82f8e5e9bcf2543 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543 2024-11-24T02:42:55,308 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:42:55,308 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:55,308 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,308 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/cf8575d7d6c4457e8049afbc61dc6062, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=36.4 K 2024-11-24T02:42:55,309 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8575d7d6c4457e8049afbc61dc6062, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732416170263 2024-11-24T02:42:55,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3fa869672e4e49a66e42d7344eb89e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732416170980 2024-11-24T02:42:55,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting acb0dc91d492422fa19713fa47601792, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:55,314 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into 79955c4d15bf4e6aa82f8e5e9bcf2543(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:55,314 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:55,315 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=13, startTime=1732416174863; duration=0sec 2024-11-24T02:42:55,315 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:55,315 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:42:55,321 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:55,322 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/e63178b4672447d4acbb51d994ba6130 is 50, key is test_row_0/C:col10/1732416172417/Put/seqid=0 2024-11-24T02:42:55,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742045_1221 (size=12949) 2024-11-24T02:42:55,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416235479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416235479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:55,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:55,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:55,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/79e1603bf1d8400a850fc9199acd2778 2024-11-24T02:42:55,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/fa868a0dbf55429fae695e2cf9bbbcb0 is 50, key is test_row_0/C:col10/1732416175121/Put/seqid=0 2024-11-24T02:42:55,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742046_1222 (size=12301) 2024-11-24T02:42:55,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,731 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/e63178b4672447d4acbb51d994ba6130 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/e63178b4672447d4acbb51d994ba6130 2024-11-24T02:42:55,738 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into e63178b4672447d4acbb51d994ba6130(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:55,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:55,738 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=13, startTime=1732416174863; duration=0sec 2024-11-24T02:42:55,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:55,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:42:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416235788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416235789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:55,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:55,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416235819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:55,820 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:55,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:55,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:55,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:55,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:55,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:42:55,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/fa868a0dbf55429fae695e2cf9bbbcb0 2024-11-24T02:42:55,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/13cc48d0e3a94ed68570171e719561f1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1 2024-11-24T02:42:55,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1, entries=150, sequenceid=296, filesize=30.5 K 2024-11-24T02:42:56,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/79e1603bf1d8400a850fc9199acd2778 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778 2024-11-24T02:42:56,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778, entries=150, sequenceid=296, filesize=12.0 K 2024-11-24T02:42:56,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/fa868a0dbf55429fae695e2cf9bbbcb0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0 2024-11-24T02:42:56,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0, entries=150, sequenceid=296, filesize=12.0 K 2024-11-24T02:42:56,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for a7378d33aaa14aba026a2abf2aaf9320 in 888ms, sequenceid=296, compaction requested=false 2024-11-24T02:42:56,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:56,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:56,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:56,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bd5a8aa0794b4e69adad06f0c5b5d0b9_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416175129/Put/seqid=0 2024-11-24T02:42:56,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742047_1223 (size=12454) 2024-11-24T02:42:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:56,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bd5a8aa0794b4e69adad06f0c5b5d0b9_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd5a8aa0794b4e69adad06f0c5b5d0b9_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/4b5b204f9a6546229f102214de212770, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:56,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/4b5b204f9a6546229f102214de212770 is 175, key is test_row_0/A:col10/1732416175129/Put/seqid=0 2024-11-24T02:42:56,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742048_1224 (size=31255) 2024-11-24T02:42:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:56,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. as already flushing 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416176335, max timestamp=1732416176353Num uniques 859; 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416176335, max timestamp=1732416176353Num uniques 859; 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416176335, max timestamp=1732416176353Num uniques 859; 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:56,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:42:56,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416236390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416236397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416236509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416236509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,545 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/4b5b204f9a6546229f102214de212770 2024-11-24T02:42:56,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/080a77df0c2840b38730c4f65fd07af6 is 50, key is test_row_0/B:col10/1732416175129/Put/seqid=0 2024-11-24T02:42:56,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742049_1225 (size=12301) 2024-11-24T02:42:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416236719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416236719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:56,955 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/080a77df0c2840b38730c4f65fd07af6 2024-11-24T02:42:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/3da3e515d2b041fe9d85322879db9606 is 50, key is test_row_0/C:col10/1732416175129/Put/seqid=0 2024-11-24T02:42:56,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742050_1226 (size=12301) 2024-11-24T02:42:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416237029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416237029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:57,367 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/3da3e515d2b041fe9d85322879db9606 2024-11-24T02:42:57,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/4b5b204f9a6546229f102214de212770 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770 2024-11-24T02:42:57,375 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770, entries=150, sequenceid=310, filesize=30.5 K 2024-11-24T02:42:57,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/080a77df0c2840b38730c4f65fd07af6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6 2024-11-24T02:42:57,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6, entries=150, sequenceid=310, filesize=12.0 K 2024-11-24T02:42:57,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/3da3e515d2b041fe9d85322879db9606 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606 2024-11-24T02:42:57,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606, entries=150, sequenceid=310, filesize=12.0 K 2024-11-24T02:42:57,385 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a7378d33aaa14aba026a2abf2aaf9320 in 1287ms, sequenceid=310, compaction requested=true 2024-11-24T02:42:57,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:57,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:57,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T02:42:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T02:42:57,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T02:42:57,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2040 sec 2024-11-24T02:42:57,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.2090 sec 2024-11-24T02:42:57,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:57,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:57,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:42:57,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d30a03a2e36a4895bd53eb31caca135e_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:57,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742051_1227 (size=17534) 2024-11-24T02:42:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416237558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416237559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416237668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416237668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49800 deadline: 1732416237839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,845 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416237879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416237879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:57,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:57,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:42:57,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d30a03a2e36a4895bd53eb31caca135e_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d30a03a2e36a4895bd53eb31caca135e_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:57,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/2e14f72ec91f4bbeb0f2258fca2835df, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:57,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/2e14f72ec91f4bbeb0f2258fca2835df is 175, key is test_row_0/A:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742052_1228 (size=48639) 2024-11-24T02:42:58,182 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-24T02:42:58,182 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:58,182 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,182 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:58,183 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-24T02:42:58,183 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:58,183 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,183 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:58,188 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-11-24T02:42:58,188 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:58,188 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,188 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416238189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416238189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:58,190 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-24T02:42:58,190 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:58,190 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,190 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:58,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:58,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/2e14f72ec91f4bbeb0f2258fca2835df 2024-11-24T02:42:58,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c5974e3752824b4d97deff2cc94317e3 is 50, key is test_row_0/B:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742053_1229 (size=12301) 2024-11-24T02:42:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49768 deadline: 1732416238699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49792 deadline: 1732416238699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:58,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c5974e3752824b4d97deff2cc94317e3 2024-11-24T02:42:58,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/480efe24e14a4e4cafd402bf014b23d4 is 50, key is test_row_0/C:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:58,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742054_1230 (size=12301) 2024-11-24T02:42:59,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:59,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:42:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49748 deadline: 1732416239209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49812 deadline: 1732416239209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:42:59,210 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:59,210 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in a7378d33aaa14aba026a2abf2aaf9320 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:59,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:42:59,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/480efe24e14a4e4cafd402bf014b23d4 2024-11-24T02:42:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/2e14f72ec91f4bbeb0f2258fca2835df as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df 2024-11-24T02:42:59,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df, entries=250, sequenceid=347, filesize=47.5 K 2024-11-24T02:42:59,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/c5974e3752824b4d97deff2cc94317e3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3 2024-11-24T02:42:59,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3, entries=150, sequenceid=347, filesize=12.0 K 2024-11-24T02:42:59,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/480efe24e14a4e4cafd402bf014b23d4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4 2024-11-24T02:42:59,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4, entries=150, sequenceid=347, filesize=12.0 K 2024-11-24T02:42:59,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for a7378d33aaa14aba026a2abf2aaf9320 in 1745ms, sequenceid=347, compaction requested=true 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:59,288 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a7378d33aaa14aba026a2abf2aaf9320:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:42:59,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:59,288 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:59,290 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:59,290 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143052 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:59,290 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/B is initiating minor compaction (all files) 2024-11-24T02:42:59,290 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/A is initiating minor compaction (all files) 2024-11-24T02:42:59,290 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/B in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:59,290 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/A in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:59,290 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/94b76ef708a64c9c927a7d4a8fb0935c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=48.7 K 2024-11-24T02:42:59,290 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=139.7 K 2024-11-24T02:42:59,290 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:59,290 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df] 2024-11-24T02:42:59,291 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b76ef708a64c9c927a7d4a8fb0935c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:59,291 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79955c4d15bf4e6aa82f8e5e9bcf2543, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:59,291 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e1603bf1d8400a850fc9199acd2778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732416173627 2024-11-24T02:42:59,291 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13cc48d0e3a94ed68570171e719561f1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732416173627 2024-11-24T02:42:59,291 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 080a77df0c2840b38730c4f65fd07af6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732416175129 2024-11-24T02:42:59,292 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5b204f9a6546229f102214de212770, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732416175129 2024-11-24T02:42:59,292 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c5974e3752824b4d97deff2cc94317e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732416176395 2024-11-24T02:42:59,292 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e14f72ec91f4bbeb0f2258fca2835df, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732416176387 2024-11-24T02:42:59,303 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:59,304 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#B#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:59,304 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/ee5b7a981a9a4cefa875eaef3ef3200f is 50, key is test_row_0/B:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:59,305 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411243642657194cb42cdb26698a0b9c197fa_a7378d33aaa14aba026a2abf2aaf9320 store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:59,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742055_1231 (size=13085) 2024-11-24T02:42:59,318 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/ee5b7a981a9a4cefa875eaef3ef3200f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ee5b7a981a9a4cefa875eaef3ef3200f 2024-11-24T02:42:59,324 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/B of a7378d33aaa14aba026a2abf2aaf9320 into ee5b7a981a9a4cefa875eaef3ef3200f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:59,324 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:59,324 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/B, priority=12, startTime=1732416179288; duration=0sec 2024-11-24T02:42:59,324 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:42:59,324 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:B 2024-11-24T02:42:59,324 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:42:59,325 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:42:59,326 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): a7378d33aaa14aba026a2abf2aaf9320/C is initiating minor compaction (all files) 2024-11-24T02:42:59,326 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a7378d33aaa14aba026a2abf2aaf9320/C in TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:42:59,326 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/e63178b4672447d4acbb51d994ba6130, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp, totalSize=48.7 K 2024-11-24T02:42:59,326 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e63178b4672447d4acbb51d994ba6130, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416172417 2024-11-24T02:42:59,326 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fa868a0dbf55429fae695e2cf9bbbcb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732416173627 2024-11-24T02:42:59,327 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da3e515d2b041fe9d85322879db9606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732416175129 2024-11-24T02:42:59,327 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 480efe24e14a4e4cafd402bf014b23d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732416176395 2024-11-24T02:42:59,333 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411243642657194cb42cdb26698a0b9c197fa_a7378d33aaa14aba026a2abf2aaf9320, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:59,333 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243642657194cb42cdb26698a0b9c197fa_a7378d33aaa14aba026a2abf2aaf9320 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:42:59,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742056_1232 (size=4469) 2024-11-24T02:42:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:42:59,339 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:42:59,343 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#C#compaction#194 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:59,343 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d7a1e6d4b59b46f897b56e694fdc5cbc is 50, key is test_row_0/C:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:59,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742057_1233 (size=13085) 2024-11-24T02:42:59,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:42:59,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:42:59,711 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-24T02:42:59,711 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-24T02:42:59,711 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:59,712 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:59,712 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:42:59,712 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:42:59,712 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:59,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:59,712 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:59,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:42:59,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:42:59,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fe25e1feea8042dc8809dd2bc4b34546_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416179709/Put/seqid=0 2024-11-24T02:42:59,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742058_1234 (size=12454) 2024-11-24T02:42:59,739 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a7378d33aaa14aba026a2abf2aaf9320#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:42:59,739 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/aa3100ddcd2a4732929d759852c2da2c is 175, key is test_row_0/A:col10/1732416177541/Put/seqid=0 2024-11-24T02:42:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742059_1235 (size=32039) 2024-11-24T02:42:59,752 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d7a1e6d4b59b46f897b56e694fdc5cbc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d7a1e6d4b59b46f897b56e694fdc5cbc 2024-11-24T02:42:59,758 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/C of a7378d33aaa14aba026a2abf2aaf9320 into d7a1e6d4b59b46f897b56e694fdc5cbc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:42:59,758 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:42:59,758 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/C, priority=12, startTime=1732416179288; duration=0sec 2024-11-24T02:42:59,758 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:42:59,758 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:C 2024-11-24T02:43:00,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:00,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fe25e1feea8042dc8809dd2bc4b34546_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fe25e1feea8042dc8809dd2bc4b34546_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:00,136 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/799c7785bdb848de9b01b0184b7e8352, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:43:00,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/799c7785bdb848de9b01b0184b7e8352 is 175, key is test_row_0/A:col10/1732416179709/Put/seqid=0 2024-11-24T02:43:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742060_1236 (size=31255) 2024-11-24T02:43:00,147 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/aa3100ddcd2a4732929d759852c2da2c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/aa3100ddcd2a4732929d759852c2da2c 2024-11-24T02:43:00,152 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a7378d33aaa14aba026a2abf2aaf9320/A of a7378d33aaa14aba026a2abf2aaf9320 into aa3100ddcd2a4732929d759852c2da2c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:00,152 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:43:00,152 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., storeName=a7378d33aaa14aba026a2abf2aaf9320/A, priority=12, startTime=1732416179288; duration=0sec 2024-11-24T02:43:00,152 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:00,152 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a7378d33aaa14aba026a2abf2aaf9320:A 2024-11-24T02:43:00,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/799c7785bdb848de9b01b0184b7e8352 2024-11-24T02:43:00,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2aee75fe51964ffc8262f42dcb0d414b is 50, key is test_row_0/B:col10/1732416179709/Put/seqid=0 2024-11-24T02:43:00,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742061_1237 (size=12301) 2024-11-24T02:43:00,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2aee75fe51964ffc8262f42dcb0d414b 2024-11-24T02:43:00,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d26f332c58a74070ae87f270a1ded382 is 50, key is test_row_0/C:col10/1732416179709/Put/seqid=0 2024-11-24T02:43:00,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742062_1238 (size=12301) 2024-11-24T02:43:01,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:43:01,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d26f332c58a74070ae87f270a1ded382 2024-11-24T02:43:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/799c7785bdb848de9b01b0184b7e8352 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/799c7785bdb848de9b01b0184b7e8352 2024-11-24T02:43:01,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/799c7785bdb848de9b01b0184b7e8352, entries=150, sequenceid=359, filesize=30.5 K 2024-11-24T02:43:01,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/2aee75fe51964ffc8262f42dcb0d414b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2aee75fe51964ffc8262f42dcb0d414b 2024-11-24T02:43:01,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2aee75fe51964ffc8262f42dcb0d414b, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:43:01,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/d26f332c58a74070ae87f270a1ded382 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d26f332c58a74070ae87f270a1ded382 2024-11-24T02:43:01,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d26f332c58a74070ae87f270a1ded382, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:43:01,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for a7378d33aaa14aba026a2abf2aaf9320 in 1679ms, sequenceid=359, compaction requested=false 2024-11-24T02:43:01,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a7378d33aaa14aba026a2abf2aaf9320: 2024-11-24T02:43:01,852 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-24T02:43:01,852 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:01,852 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:01,852 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:01,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:09,253 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-24T02:43:09,253 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:09,253 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,253 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:09,282 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-24T02:43:09,282 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:09,282 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,282 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2882 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2801 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1222 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3666 rows 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-24T02:43:09,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3656 rows 2024-11-24T02:43:09,284 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:43:09,284 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:43:09,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:09,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:09,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:43:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:09,289 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416189289"}]},"ts":"1732416189289"} 2024-11-24T02:43:09,290 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:43:09,290 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:43:09,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:43:09,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, UNASSIGN}] 2024-11-24T02:43:09,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, UNASSIGN 2024-11-24T02:43:09,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:09,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, UNASSIGN because future has completed 2024-11-24T02:43:09,294 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:43:09,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:09,447 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:09,447 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:43:09,447 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing a7378d33aaa14aba026a2abf2aaf9320, disabling compactions & flushes 2024-11-24T02:43:09,447 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:43:09,448 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:43:09,448 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. after waiting 0 ms 2024-11-24T02:43:09,448 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:43:09,448 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing a7378d33aaa14aba026a2abf2aaf9320 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T02:43:09,448 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=A 2024-11-24T02:43:09,448 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:09,449 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=B 2024-11-24T02:43:09,449 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:09,449 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a7378d33aaa14aba026a2abf2aaf9320, store=C 2024-11-24T02:43:09,449 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:09,461 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245ea8c6eecb0441fba3b74016e84cc0ae_a7378d33aaa14aba026a2abf2aaf9320 is 50, key is test_row_0/A:col10/1732416189250/Put/seqid=0 2024-11-24T02:43:09,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742063_1239 (size=9914) 2024-11-24T02:43:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:09,866 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:09,875 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245ea8c6eecb0441fba3b74016e84cc0ae_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245ea8c6eecb0441fba3b74016e84cc0ae_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:09,877 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1b9e427ca83143c09a8f1da4a32cbdb5, store: [table=TestAcidGuarantees family=A region=a7378d33aaa14aba026a2abf2aaf9320] 2024-11-24T02:43:09,877 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1b9e427ca83143c09a8f1da4a32cbdb5 is 175, key is test_row_0/A:col10/1732416189250/Put/seqid=0 2024-11-24T02:43:09,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742064_1240 (size=22561) 2024-11-24T02:43:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:10,283 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1b9e427ca83143c09a8f1da4a32cbdb5 2024-11-24T02:43:10,323 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/85d8be66b3a34ec1bce81449dadb6d8c is 50, key is test_row_0/B:col10/1732416189250/Put/seqid=0 2024-11-24T02:43:10,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742065_1241 (size=9857) 2024-11-24T02:43:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:10,728 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/85d8be66b3a34ec1bce81449dadb6d8c 2024-11-24T02:43:10,742 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/63d3d87028e04953a193b0c8243458ee is 50, key is test_row_0/C:col10/1732416189250/Put/seqid=0 2024-11-24T02:43:10,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742066_1242 (size=9857) 2024-11-24T02:43:11,147 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/63d3d87028e04953a193b0c8243458ee 2024-11-24T02:43:11,151 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/A/1b9e427ca83143c09a8f1da4a32cbdb5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1b9e427ca83143c09a8f1da4a32cbdb5 2024-11-24T02:43:11,155 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1b9e427ca83143c09a8f1da4a32cbdb5, entries=100, sequenceid=367, filesize=22.0 K 2024-11-24T02:43:11,156 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/B/85d8be66b3a34ec1bce81449dadb6d8c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/85d8be66b3a34ec1bce81449dadb6d8c 2024-11-24T02:43:11,160 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/85d8be66b3a34ec1bce81449dadb6d8c, entries=100, sequenceid=367, filesize=9.6 K 2024-11-24T02:43:11,161 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/.tmp/C/63d3d87028e04953a193b0c8243458ee as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/63d3d87028e04953a193b0c8243458ee 2024-11-24T02:43:11,164 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/63d3d87028e04953a193b0c8243458ee, entries=100, sequenceid=367, filesize=9.6 K 2024-11-24T02:43:11,165 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a7378d33aaa14aba026a2abf2aaf9320 in 1717ms, sequenceid=367, compaction requested=true 2024-11-24T02:43:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df] to archive 2024-11-24T02:43:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fc937456e1e6454590b72345995c1588 2024-11-24T02:43:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1f08984d3b70458dbfed12e0d295d4d6 2024-11-24T02:43:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f4f6b69ba5bb4ba380d2c394aa802eac 2024-11-24T02:43:11,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/b0a3e9f8719346019ee19eb033340f2b 2024-11-24T02:43:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/64ce5545601c409ba8c8200001589b4c 2024-11-24T02:43:11,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/d5908851893e4b9b94a02e6d5ae1cc55 2024-11-24T02:43:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/cd7c7f8b3e814fca8d63c1a60901e96d 2024-11-24T02:43:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/fff2487302a441349dc6a3a1365dd40c 2024-11-24T02:43:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/f0f19d6869b6448eb02abc580101d4ec 2024-11-24T02:43:11,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1d6417d336884683a0d57991339becb8 2024-11-24T02:43:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1c51fbdca99f4c67bca3d5a24e4fc3f0 2024-11-24T02:43:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c8a09aedc4084161a8f71a55f7a0e91d 2024-11-24T02:43:11,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/c4536d1bbe0541ff984063ebe35d3795 2024-11-24T02:43:11,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/6580296930ca4611a8438693df702fea 2024-11-24T02:43:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/014144b6c04242ef89e20a022e0eb54e 2024-11-24T02:43:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3975e8fc399d47c19f17b65a08356116 2024-11-24T02:43:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/3f5ff48529294856b1bb0b35f6caf3e6 2024-11-24T02:43:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/639abcc7d2da416fa0569c62d745dca6 2024-11-24T02:43:11,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/8232babd89c64272bea70df7ec9a0eae 2024-11-24T02:43:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/79955c4d15bf4e6aa82f8e5e9bcf2543 2024-11-24T02:43:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/13cc48d0e3a94ed68570171e719561f1 2024-11-24T02:43:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/4b5b204f9a6546229f102214de212770 2024-11-24T02:43:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/2e14f72ec91f4bbeb0f2258fca2835df 2024-11-24T02:43:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24f03333c4b24620bee87826250b31cd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/257c968aa9c84c13ae9d373a8d095cca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24d3a78af3c643c898891a0f123c9d5d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db1a1b393db148ff8293ace8d575f4e9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/6ae5f9981a284971b11f234154f09bab, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/94b76ef708a64c9c927a7d4a8fb0935c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3] to archive 2024-11-24T02:43:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:11,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c53467796bfe42048bcee4b9680c8fc3 2024-11-24T02:43:11,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/673a746c6aca495397a5e9f370d28297 2024-11-24T02:43:11,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24f03333c4b24620bee87826250b31cd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24f03333c4b24620bee87826250b31cd 2024-11-24T02:43:11,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/4adce482827b436187d3e63315267f25 2024-11-24T02:43:11,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db4592e8fbc543099ae98b96fe20d20e 2024-11-24T02:43:11,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/257c968aa9c84c13ae9d373a8d095cca to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/257c968aa9c84c13ae9d373a8d095cca 2024-11-24T02:43:11,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2b2382a46c794ab7b402b9003112ea34 2024-11-24T02:43:11,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2f5d7b4f239b43e99098ff8b350f3ec7 2024-11-24T02:43:11,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24d3a78af3c643c898891a0f123c9d5d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/24d3a78af3c643c898891a0f123c9d5d 2024-11-24T02:43:11,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/362350ccc6ef49ef80f71a7242b39586 2024-11-24T02:43:11,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/a134886ed878425696c47959e02837a8 2024-11-24T02:43:11,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ebc19c304bd1441ea3a88d685b81b8ac 2024-11-24T02:43:11,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db1a1b393db148ff8293ace8d575f4e9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/db1a1b393db148ff8293ace8d575f4e9 2024-11-24T02:43:11,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/22744c1c970e4c35b42d83a9c46af5a9 2024-11-24T02:43:11,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/679c755f0dc94195aae41d95cd68ff0c 2024-11-24T02:43:11,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/6ae5f9981a284971b11f234154f09bab to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/6ae5f9981a284971b11f234154f09bab 2024-11-24T02:43:11,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/13330cf633b04d59a4be0f31b6212f6f 2024-11-24T02:43:11,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/7a35e0a2c5404b7ca9eb66446508b2dc 2024-11-24T02:43:11,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/94b76ef708a64c9c927a7d4a8fb0935c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/94b76ef708a64c9c927a7d4a8fb0935c 2024-11-24T02:43:11,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/3c80b4fb3dd24afd961148a75ff572ae 2024-11-24T02:43:11,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/79e1603bf1d8400a850fc9199acd2778 2024-11-24T02:43:11,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/080a77df0c2840b38730c4f65fd07af6 2024-11-24T02:43:11,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/c5974e3752824b4d97deff2cc94317e3 2024-11-24T02:43:11,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/c8fde89bd2af44b4863403db1326a50b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/72f66cfff09e4d0280911d6786899615, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/5395557d7911421f86c1a144c97b7a03, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fca492b68ab2450c85dd440ae0a855b4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/cf8575d7d6c4457e8049afbc61dc6062, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/e63178b4672447d4acbb51d994ba6130, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4] to archive 2024-11-24T02:43:11,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:11,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/6e3cc7c33ed84fd6979353ce47fb72ef 2024-11-24T02:43:11,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/8ca49f5ff41242ff8e21e49e3171c46e 2024-11-24T02:43:11,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/c8fde89bd2af44b4863403db1326a50b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/c8fde89bd2af44b4863403db1326a50b 2024-11-24T02:43:11,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/436b49b61c9f45eb9c4f542dbbf829e9 2024-11-24T02:43:11,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/b32b1bc6f16348009daaf9c7600ddb5f 2024-11-24T02:43:11,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/72f66cfff09e4d0280911d6786899615 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/72f66cfff09e4d0280911d6786899615 2024-11-24T02:43:11,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/31a45940b2bd43609ad3dbf654d0e62c 2024-11-24T02:43:11,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/bef6d98ec16645a08ea48394f9af9240 2024-11-24T02:43:11,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/5395557d7911421f86c1a144c97b7a03 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/5395557d7911421f86c1a144c97b7a03 2024-11-24T02:43:11,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/10a8c50017524f4d9525a3b6a2de5d3c 2024-11-24T02:43:11,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/f07ba1e6245147c5b0a5e413ec93e9b8 2024-11-24T02:43:11,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/7e1863ce281246ebab29f615ca2712c4 2024-11-24T02:43:11,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fca492b68ab2450c85dd440ae0a855b4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fca492b68ab2450c85dd440ae0a855b4 2024-11-24T02:43:11,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/735c11e0e7eb45beafaea9bd3fbd4330 2024-11-24T02:43:11,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d3125cff2d174f378e76347ba8265f0c 2024-11-24T02:43:11,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/cf8575d7d6c4457e8049afbc61dc6062 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/cf8575d7d6c4457e8049afbc61dc6062 2024-11-24T02:43:11,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/017a6420e4174c339948173680979833 2024-11-24T02:43:11,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/0f3fa869672e4e49a66e42d7344eb89e 2024-11-24T02:43:11,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/e63178b4672447d4acbb51d994ba6130 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/e63178b4672447d4acbb51d994ba6130 2024-11-24T02:43:11,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/acb0dc91d492422fa19713fa47601792 2024-11-24T02:43:11,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/fa868a0dbf55429fae695e2cf9bbbcb0 2024-11-24T02:43:11,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/3da3e515d2b041fe9d85322879db9606 2024-11-24T02:43:11,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/480efe24e14a4e4cafd402bf014b23d4 2024-11-24T02:43:11,244 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-24T02:43:11,245 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. 2024-11-24T02:43:11,245 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for a7378d33aaa14aba026a2abf2aaf9320: Waiting for close lock at 1732416189447Running coprocessor pre-close hooks at 1732416189447Disabling compacts and flushes for region at 1732416189447Disabling writes for close at 1732416189448 (+1 ms)Obtaining lock to block concurrent updates at 1732416189448Preparing flush snapshotting stores in a7378d33aaa14aba026a2abf2aaf9320 at 1732416189448Finished memstore snapshotting TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732416189449 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320. at 1732416189450 (+1 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/A: creating writer at 1732416189450Flushing mob file a7378d33aaa14aba026a2abf2aaf9320/A: appending metadata at 1732416189460 (+10 ms)Flushing mob file a7378d33aaa14aba026a2abf2aaf9320/A: closing flushed file at 1732416189460Flushing a7378d33aaa14aba026a2abf2aaf9320/A: appending metadata at 1732416189877 (+417 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/A: closing flushed file at 1732416189877Flushing a7378d33aaa14aba026a2abf2aaf9320/B: creating writer at 1732416190316 (+439 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/B: appending metadata at 1732416190322 (+6 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/B: closing flushed file at 1732416190322Flushing a7378d33aaa14aba026a2abf2aaf9320/C: creating writer at 1732416190737 (+415 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/C: appending metadata at 1732416190741 (+4 ms)Flushing a7378d33aaa14aba026a2abf2aaf9320/C: closing flushed file at 1732416190741Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24a566a: reopening flushed file at 1732416191151 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9d0685d: reopening flushed file at 1732416191155 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be4d205: reopening flushed file at 1732416191160 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a7378d33aaa14aba026a2abf2aaf9320 in 1717ms, sequenceid=367, compaction requested=true at 1732416191165 (+5 ms)Writing region close event to WAL at 1732416191242 (+77 ms)Running coprocessor post-close hooks at 1732416191245 (+3 ms)Closed at 1732416191245 2024-11-24T02:43:11,247 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,247 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=a7378d33aaa14aba026a2abf2aaf9320, regionState=CLOSED 2024-11-24T02:43:11,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:11,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T02:43:11,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure a7378d33aaa14aba026a2abf2aaf9320, server=7c69a60bd8f6,34251,1732416122955 in 1.9550 sec 2024-11-24T02:43:11,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-24T02:43:11,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a7378d33aaa14aba026a2abf2aaf9320, UNASSIGN in 1.9580 sec 2024-11-24T02:43:11,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T02:43:11,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-24T02:43:11,254 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416191254"}]},"ts":"1732416191254"} 2024-11-24T02:43:11,255 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:43:11,255 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:43:11,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9700 sec 2024-11-24T02:43:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:43:11,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:11,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:43:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,446 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:43:11,447 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits] 2024-11-24T02:43:11,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1b9e427ca83143c09a8f1da4a32cbdb5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/1b9e427ca83143c09a8f1da4a32cbdb5 2024-11-24T02:43:11,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/799c7785bdb848de9b01b0184b7e8352 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/799c7785bdb848de9b01b0184b7e8352 2024-11-24T02:43:11,464 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/aa3100ddcd2a4732929d759852c2da2c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/A/aa3100ddcd2a4732929d759852c2da2c 2024-11-24T02:43:11,467 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2aee75fe51964ffc8262f42dcb0d414b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/2aee75fe51964ffc8262f42dcb0d414b 2024-11-24T02:43:11,468 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/85d8be66b3a34ec1bce81449dadb6d8c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/85d8be66b3a34ec1bce81449dadb6d8c 2024-11-24T02:43:11,469 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ee5b7a981a9a4cefa875eaef3ef3200f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/B/ee5b7a981a9a4cefa875eaef3ef3200f 2024-11-24T02:43:11,472 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/63d3d87028e04953a193b0c8243458ee to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/63d3d87028e04953a193b0c8243458ee 2024-11-24T02:43:11,473 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d26f332c58a74070ae87f270a1ded382 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d26f332c58a74070ae87f270a1ded382 2024-11-24T02:43:11,474 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d7a1e6d4b59b46f897b56e694fdc5cbc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/C/d7a1e6d4b59b46f897b56e694fdc5cbc 2024-11-24T02:43:11,476 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits/370.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320/recovered.edits/370.seqid 2024-11-24T02:43:11,477 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,477 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:43:11,477 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:43:11,478 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:43:11,482 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240071528dd6374a779ae0073c37ef811b_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240071528dd6374a779ae0073c37ef811b_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,483 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112406941c6ccc10416d9786d59cb738434e_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112406941c6ccc10416d9786d59cb738434e_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,484 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112427cfcf09341d4b62af133da0a03ed441_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112427cfcf09341d4b62af133da0a03ed441_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,485 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124335fd87a815e4e099f0f2103c17d2b01_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124335fd87a815e4e099f0f2103c17d2b01_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,486 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437d4a0f87e0640d4868c2176e54cfb02_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437d4a0f87e0640d4868c2176e54cfb02_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,487 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458bfaafc30c64ab39fdc31f1b3d7a5e7_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458bfaafc30c64ab39fdc31f1b3d7a5e7_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,488 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245ea8c6eecb0441fba3b74016e84cc0ae_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245ea8c6eecb0441fba3b74016e84cc0ae_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,489 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112460d51249a16f4456ae23bb2ecf3516dc_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112460d51249a16f4456ae23bb2ecf3516dc_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,489 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a47bd5c3e9ac42598e0f1a2d33172fef_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a47bd5c3e9ac42598e0f1a2d33172fef_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,490 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb7cebddfc414666a659fe4ca7aeb5d6_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb7cebddfc414666a659fe4ca7aeb5d6_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,491 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd5a8aa0794b4e69adad06f0c5b5d0b9_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd5a8aa0794b4e69adad06f0c5b5d0b9_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,492 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f6b79b122d40dfb4eb99bd5033924d_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f6b79b122d40dfb4eb99bd5033924d_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,493 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d30a03a2e36a4895bd53eb31caca135e_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d30a03a2e36a4895bd53eb31caca135e_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,494 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de92016413814986be3521b223a14d53_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de92016413814986be3521b223a14d53_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,495 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ebc9b1d4c3eb4f95b69a028e42325960_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ebc9b1d4c3eb4f95b69a028e42325960_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,496 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9b2381afb0c424695630632e185df84_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9b2381afb0c424695630632e185df84_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,496 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc25a2630b8a456287c5d2ad3c88b8e7_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc25a2630b8a456287c5d2ad3c88b8e7_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,497 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fcda7cb7b9ed4f8c9128ebc7403c31f4_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fcda7cb7b9ed4f8c9128ebc7403c31f4_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,498 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fe25e1feea8042dc8809dd2bc4b34546_a7378d33aaa14aba026a2abf2aaf9320 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fe25e1feea8042dc8809dd2bc4b34546_a7378d33aaa14aba026a2abf2aaf9320 2024-11-24T02:43:11,499 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:43:11,501 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,504 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:43:11,506 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:43:11,507 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,507 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:43:11,508 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416191508"}]},"ts":"9223372036854775807"} 2024-11-24T02:43:11,510 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:43:11,510 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a7378d33aaa14aba026a2abf2aaf9320, NAME => 'TestAcidGuarantees,,1732416154761.a7378d33aaa14aba026a2abf2aaf9320.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:43:11,510 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:43:11,510 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416191510"}]},"ts":"9223372036854775807"} 2024-11-24T02:43:11,512 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:43:11,513 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-24T02:43:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:43:11,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:43:11,559 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:11,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 419) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10286 (was 10690) 2024-11-24T02:43:11,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=10286 2024-11-24T02:43:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:43:11,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:43:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:11,583 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:43:11,583 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:11,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-24T02:43:11,584 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:43:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:11,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742067_1243 (size=1061) 2024-11-24T02:43:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:11,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:11,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:43:11,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742068_1244 (size=53) 2024-11-24T02:43:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:12,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:43:12,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f40cba6b6b9bf91dffe1d7372b246258, disabling compactions & flushes 2024-11-24T02:43:12,401 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,401 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. after waiting 0 ms 2024-11-24T02:43:12,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f40cba6b6b9bf91dffe1d7372b246258: Waiting for close lock at 1732416192401Disabling compacts and flushes for region at 1732416192401Disabling writes for close at 1732416192402 (+1 ms)Writing region close event to WAL at 1732416192402Closed at 1732416192402 2024-11-24T02:43:12,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:43:12,406 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416192405"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416192405"}]},"ts":"1732416192405"} 2024-11-24T02:43:12,408 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:43:12,409 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:43:12,410 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416192409"}]},"ts":"1732416192409"} 2024-11-24T02:43:12,411 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:43:12,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, ASSIGN}] 2024-11-24T02:43:12,413 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, ASSIGN 2024-11-24T02:43:12,414 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:43:12,518 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:43:12,565 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f40cba6b6b9bf91dffe1d7372b246258, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:12,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, ASSIGN because future has completed 2024-11-24T02:43:12,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:12,733 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,733 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:43:12,734 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,734 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:43:12,734 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,734 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,736 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,737 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:12,737 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f40cba6b6b9bf91dffe1d7372b246258 columnFamilyName A 2024-11-24T02:43:12,737 DEBUG [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:12,738 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(327): Store=f40cba6b6b9bf91dffe1d7372b246258/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:12,738 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,739 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:12,740 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f40cba6b6b9bf91dffe1d7372b246258 columnFamilyName B 2024-11-24T02:43:12,740 DEBUG [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:12,740 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(327): Store=f40cba6b6b9bf91dffe1d7372b246258/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:12,740 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,742 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:12,742 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f40cba6b6b9bf91dffe1d7372b246258 columnFamilyName C 2024-11-24T02:43:12,742 DEBUG [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:12,743 INFO [StoreOpener-f40cba6b6b9bf91dffe1d7372b246258-1 {}] regionserver.HStore(327): Store=f40cba6b6b9bf91dffe1d7372b246258/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:12,743 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,743 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,744 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,744 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,745 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,745 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,746 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:43:12,747 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,749 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:43:12,749 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened f40cba6b6b9bf91dffe1d7372b246258; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59136018, jitterRate=-0.11880466341972351}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:43:12,749 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:12,750 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for f40cba6b6b9bf91dffe1d7372b246258: Running coprocessor pre-open hook at 1732416192734Writing region info on filesystem at 1732416192734Initializing all the Stores at 1732416192735 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416192735Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416192735Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416192735Cleaning up temporary data from old regions at 1732416192745 (+10 ms)Running coprocessor post-open hooks at 1732416192749 (+4 ms)Region opened successfully at 1732416192750 (+1 ms) 2024-11-24T02:43:12,751 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., pid=61, masterSystemTime=1732416192725 2024-11-24T02:43:12,752 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,752 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:12,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=f40cba6b6b9bf91dffe1d7372b246258, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:12,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:12,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T02:43:12,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 in 184 msec 2024-11-24T02:43:12,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-24T02:43:12,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, ASSIGN in 344 msec 2024-11-24T02:43:12,758 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:43:12,758 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416192758"}]},"ts":"1732416192758"} 2024-11-24T02:43:12,760 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:43:12,760 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:43:12,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-24T02:43:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:43:13,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:13,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba7ffab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,736 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2d2313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,740 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519e4a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,746 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:43:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:43:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@9cb994d 2024-11-24T02:43:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:43:13,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:43:13,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11961dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35438b71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64dc3a64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48345386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,755 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c46979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6675bfb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ddee8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b871e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,761 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bee2806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5138abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,764 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aaa39b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dc0396, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd83e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14991ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2847dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29529392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,774 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3025e9bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289dbcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7341fa9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b9d05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@832215d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,778 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@163b2c10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f98a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478bceb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,781 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78220aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5ac36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63eadf7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:13,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:13,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc14de7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:13,785 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:13,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfd4b91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:13,786 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,787 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,786 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,787 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,787 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,788 DEBUG [Thread-1135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-24T02:43:13,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:13,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:13,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,791 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:13,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:13,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:43:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:13,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:13,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a1c4876518094b96aa829cc77f20f4c1 is 50, key is test_row_0/A:col10/1732416193798/Put/seqid=0 2024-11-24T02:43:13,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742069_1245 (size=14341) 2024-11-24T02:43:13,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416253857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416253857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416253857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416253859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416253864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:13,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:13,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:13,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416253969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416253969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416253970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416253972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416253979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416254179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416254189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a1c4876518094b96aa829cc77f20f4c1 2024-11-24T02:43:14,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:14,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b4662286f08741d18748842f68c191b9 is 50, key is test_row_0/B:col10/1732416193798/Put/seqid=0 2024-11-24T02:43:14,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742070_1246 (size=12001) 2024-11-24T02:43:14,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b4662286f08741d18748842f68c191b9 2024-11-24T02:43:14,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5fd2c0951475421182e2b88c0cc17b1f is 50, key is test_row_0/C:col10/1732416193798/Put/seqid=0 2024-11-24T02:43:14,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742071_1247 (size=12001) 2024-11-24T02:43:14,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:14,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416254488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416254488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416254488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416254489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416254499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:14,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:14,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5fd2c0951475421182e2b88c0cc17b1f 2024-11-24T02:43:14,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a1c4876518094b96aa829cc77f20f4c1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1 2024-11-24T02:43:14,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1, entries=200, sequenceid=14, filesize=14.0 K 2024-11-24T02:43:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b4662286f08741d18748842f68c191b9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9 2024-11-24T02:43:14,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T02:43:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:14,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5fd2c0951475421182e2b88c0cc17b1f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f 2024-11-24T02:43:14,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T02:43:14,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f40cba6b6b9bf91dffe1d7372b246258 in 915ms, sequenceid=14, compaction requested=false 2024-11-24T02:43:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:43:14,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:14,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:14,865 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:14,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:14,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ac3fc9c6fea44091b10b7f054a6adc2d is 50, key is test_row_0/A:col10/1732416193829/Put/seqid=0 2024-11-24T02:43:14,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742072_1248 (size=12001) 2024-11-24T02:43:14,879 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ac3fc9c6fea44091b10b7f054a6adc2d 2024-11-24T02:43:14,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/bc12a2c4f77a40448e3d850023c4cfa2 is 50, key is test_row_0/B:col10/1732416193829/Put/seqid=0 2024-11-24T02:43:14,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742073_1249 (size=12001) 2024-11-24T02:43:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:15,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416255007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416255008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416255009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416255009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416255011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416255119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416255119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416255119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416255119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/bc12a2c4f77a40448e3d850023c4cfa2 2024-11-24T02:43:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416255329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416255329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416255329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416255329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/9ef5065d4c9140798446c6b66224b913 is 50, key is test_row_0/C:col10/1732416193829/Put/seqid=0 2024-11-24T02:43:15,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742074_1250 (size=12001) 2024-11-24T02:43:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416255639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416255639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416255639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416255639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:15,782 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/9ef5065d4c9140798446c6b66224b913 2024-11-24T02:43:15,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ac3fc9c6fea44091b10b7f054a6adc2d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d 2024-11-24T02:43:15,792 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T02:43:15,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/bc12a2c4f77a40448e3d850023c4cfa2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2 2024-11-24T02:43:15,798 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T02:43:15,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/9ef5065d4c9140798446c6b66224b913 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913 2024-11-24T02:43:15,803 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T02:43:15,804 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f40cba6b6b9bf91dffe1d7372b246258 in 939ms, sequenceid=33, compaction requested=false 2024-11-24T02:43:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:15,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-24T02:43:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-24T02:43:15,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T02:43:15,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0130 sec 2024-11-24T02:43:15,809 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0220 sec 2024-11-24T02:43:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:43:15,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:15,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T02:43:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:15,941 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:15,942 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:15,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:16,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:16,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/48888cf52db84efb86a764d7b81c36a0 is 50, key is test_row_0/A:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:16,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742075_1251 (size=14341) 2024-11-24T02:43:16,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416256092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416256149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416256149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416256149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416256149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416256199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:16,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416256409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/48888cf52db84efb86a764d7b81c36a0 2024-11-24T02:43:16,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f221e96a1f094b8aa783bfce4e0dab77 is 50, key is test_row_0/B:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742076_1252 (size=12001) 2024-11-24T02:43:16,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f221e96a1f094b8aa783bfce4e0dab77 2024-11-24T02:43:16,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5e9e5673ddad45ec93765f6a52a369fd is 50, key is test_row_0/C:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742077_1253 (size=12001) 2024-11-24T02:43:16,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:16,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:16,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416256719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:16,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:16,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:43:16,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:16,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5e9e5673ddad45ec93765f6a52a369fd 2024-11-24T02:43:16,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/48888cf52db84efb86a764d7b81c36a0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0 2024-11-24T02:43:16,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0, entries=200, sequenceid=47, filesize=14.0 K 2024-11-24T02:43:16,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f221e96a1f094b8aa783bfce4e0dab77 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77 2024-11-24T02:43:16,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T02:43:16,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/5e9e5673ddad45ec93765f6a52a369fd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd 2024-11-24T02:43:16,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd, entries=150, sequenceid=47, filesize=11.7 K 2024-11-24T02:43:16,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f40cba6b6b9bf91dffe1d7372b246258 in 874ms, sequenceid=47, compaction requested=true 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:16,906 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:16,906 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:16,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:16,911 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:16,911 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:16,911 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,911 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=39.7 K 2024-11-24T02:43:16,912 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:16,912 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:16,912 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,912 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.2 K 2024-11-24T02:43:16,912 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c4876518094b96aa829cc77f20f4c1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416193798 2024-11-24T02:43:16,913 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b4662286f08741d18748842f68c191b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416193798 2024-11-24T02:43:16,913 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac3fc9c6fea44091b10b7f054a6adc2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732416193829 2024-11-24T02:43:16,913 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting bc12a2c4f77a40448e3d850023c4cfa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732416193829 2024-11-24T02:43:16,914 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48888cf52db84efb86a764d7b81c36a0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:16,914 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f221e96a1f094b8aa783bfce4e0dab77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:16,922 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:16,922 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/3fd21a7d34fe49708de85bcc9f8914fe is 50, key is test_row_0/A:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,924 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#211 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:16,925 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/1ee41f9c639345dcbb20e8c1f17cdfbf is 50, key is test_row_0/B:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742078_1254 (size=12104) 2024-11-24T02:43:16,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/3fd21a7d34fe49708de85bcc9f8914fe as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/3fd21a7d34fe49708de85bcc9f8914fe 2024-11-24T02:43:16,953 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into 3fd21a7d34fe49708de85bcc9f8914fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:16,953 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:16,953 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416196906; duration=0sec 2024-11-24T02:43:16,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:16,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:16,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:16,956 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:16,956 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:16,956 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:16,956 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.2 K 2024-11-24T02:43:16,957 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd2c0951475421182e2b88c0cc17b1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732416193798 2024-11-24T02:43:16,957 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef5065d4c9140798446c6b66224b913, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732416193829 2024-11-24T02:43:16,959 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9e5673ddad45ec93765f6a52a369fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742079_1255 (size=12104) 2024-11-24T02:43:16,969 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:16,969 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/588ee4f39a094b6091ea535d2565f58d is 50, key is test_row_0/C:col10/1732416195007/Put/seqid=0 2024-11-24T02:43:16,974 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/1ee41f9c639345dcbb20e8c1f17cdfbf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1ee41f9c639345dcbb20e8c1f17cdfbf 2024-11-24T02:43:16,983 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into 1ee41f9c639345dcbb20e8c1f17cdfbf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:16,983 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:16,983 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416196906; duration=0sec 2024-11-24T02:43:16,983 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:16,983 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:16,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742080_1256 (size=12104) 2024-11-24T02:43:17,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:43:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:17,016 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/acec304c5c50433da42228ad7602b903 is 50, key is test_row_0/A:col10/1732416196071/Put/seqid=0 2024-11-24T02:43:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742081_1257 (size=12001) 2024-11-24T02:43:17,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/acec304c5c50433da42228ad7602b903 2024-11-24T02:43:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/e89cbd83545d42d293ad55d0f0535768 is 50, key is test_row_0/B:col10/1732416196071/Put/seqid=0 2024-11-24T02:43:17,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742082_1258 (size=12001) 2024-11-24T02:43:17,068 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/e89cbd83545d42d293ad55d0f0535768 2024-11-24T02:43:17,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/a6c6bf39fc804cd8922d4f6cb2787bc7 is 50, key is test_row_0/C:col10/1732416196071/Put/seqid=0 2024-11-24T02:43:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742083_1259 (size=12001) 2024-11-24T02:43:17,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/a6c6bf39fc804cd8922d4f6cb2787bc7 2024-11-24T02:43:17,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/acec304c5c50433da42228ad7602b903 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903 2024-11-24T02:43:17,110 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903, entries=150, sequenceid=67, filesize=11.7 K 2024-11-24T02:43:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/e89cbd83545d42d293ad55d0f0535768 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768 2024-11-24T02:43:17,118 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768, entries=150, sequenceid=67, filesize=11.7 K 2024-11-24T02:43:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/a6c6bf39fc804cd8922d4f6cb2787bc7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7 2024-11-24T02:43:17,125 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7, entries=150, sequenceid=67, filesize=11.7 K 2024-11-24T02:43:17,126 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for f40cba6b6b9bf91dffe1d7372b246258 in 110ms, sequenceid=67, compaction requested=false 2024-11-24T02:43:17,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:17,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:17,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T02:43:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T02:43:17,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T02:43:17,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1850 sec 2024-11-24T02:43:17,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1900 sec 2024-11-24T02:43:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:17,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:17,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:17,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/de7cd734cb794a6e836e279ab2fbc0e9 is 50, key is test_row_0/A:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:17,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742084_1260 (size=12001) 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416197180, max timestamp=1732416197210Num uniques 859; 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416197180, max timestamp=1732416197210Num uniques 859; 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416197180, max timestamp=1732416197210Num uniques 859; 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:43:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:43:17,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416257232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416257243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416257244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416257245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416257244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416257349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416257359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416257359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416257360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416257360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,394 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/588ee4f39a094b6091ea535d2565f58d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/588ee4f39a094b6091ea535d2565f58d 2024-11-24T02:43:17,398 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 588ee4f39a094b6091ea535d2565f58d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:17,398 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:17,398 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416196906; duration=0sec 2024-11-24T02:43:17,398 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:17,398 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416257559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416257568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416257569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416257569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416257569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/de7cd734cb794a6e836e279ab2fbc0e9 2024-11-24T02:43:17,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/278c2339d9e944f0a72b7db1420e71bb is 50, key is test_row_0/B:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742085_1261 (size=12001) 2024-11-24T02:43:17,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416257869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416257878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416257879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416257879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416257879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/278c2339d9e944f0a72b7db1420e71bb 2024-11-24T02:43:18,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3e235f0479bf4d09bf786d5c2b5149c8 is 50, key is test_row_0/C:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:18,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742086_1262 (size=12001) 2024-11-24T02:43:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:43:18,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:18,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T02:43:18,091 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:18,091 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:18,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:18,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T02:43:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416258379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416258389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416258389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416258389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:18,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416258390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:18,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T02:43:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:18,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3e235f0479bf4d09bf786d5c2b5149c8 2024-11-24T02:43:18,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/de7cd734cb794a6e836e279ab2fbc0e9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9 2024-11-24T02:43:18,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9, entries=150, sequenceid=77, filesize=11.7 K 2024-11-24T02:43:18,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/278c2339d9e944f0a72b7db1420e71bb as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb 2024-11-24T02:43:18,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb, entries=150, sequenceid=77, filesize=11.7 K 2024-11-24T02:43:18,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3e235f0479bf4d09bf786d5c2b5149c8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8 2024-11-24T02:43:18,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8, entries=150, sequenceid=77, filesize=11.7 K 2024-11-24T02:43:18,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f40cba6b6b9bf91dffe1d7372b246258 in 1264ms, sequenceid=77, compaction requested=true 2024-11-24T02:43:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:18,441 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:18,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:18,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:18,442 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:18,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:18,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:18,443 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:18,443 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:18,443 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,443 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1ee41f9c639345dcbb20e8c1f17cdfbf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.3 K 2024-11-24T02:43:18,443 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:18,443 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:18,443 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,443 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/3fd21a7d34fe49708de85bcc9f8914fe, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.3 K 2024-11-24T02:43:18,444 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ee41f9c639345dcbb20e8c1f17cdfbf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:18,444 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd21a7d34fe49708de85bcc9f8914fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:18,444 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e89cbd83545d42d293ad55d0f0535768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732416196071 2024-11-24T02:43:18,444 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting acec304c5c50433da42228ad7602b903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732416196071 2024-11-24T02:43:18,445 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting de7cd734cb794a6e836e279ab2fbc0e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:18,445 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 278c2339d9e944f0a72b7db1420e71bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:18,454 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:18,455 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/c1b505690a634d7b9281d397188018ff is 50, key is test_row_0/A:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:18,456 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#220 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:18,456 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/c349040bad954f909ba0988def9eec79 is 50, key is test_row_0/B:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742088_1264 (size=12207) 2024-11-24T02:43:18,467 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/c349040bad954f909ba0988def9eec79 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c349040bad954f909ba0988def9eec79 2024-11-24T02:43:18,472 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into c349040bad954f909ba0988def9eec79(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:18,472 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:18,472 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416198441; duration=0sec 2024-11-24T02:43:18,472 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:18,472 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:18,472 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:18,473 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:18,473 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:18,473 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,474 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/588ee4f39a094b6091ea535d2565f58d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.3 K 2024-11-24T02:43:18,474 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 588ee4f39a094b6091ea535d2565f58d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732416195004 2024-11-24T02:43:18,474 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c6bf39fc804cd8922d4f6cb2787bc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1732416196071 2024-11-24T02:43:18,475 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e235f0479bf4d09bf786d5c2b5149c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:18,482 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:18,483 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/c14d017b52be438794227cdad0f65ab0 is 50, key is test_row_0/C:col10/1732416197176/Put/seqid=0 2024-11-24T02:43:18,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742087_1263 (size=12207) 2024-11-24T02:43:18,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742089_1265 (size=12207) 2024-11-24T02:43:18,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T02:43:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:18,549 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T02:43:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:18,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a012f00c66d644a4a87a575b35b4a764 is 50, key is test_row_0/A:col10/1732416197230/Put/seqid=0 2024-11-24T02:43:18,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742090_1266 (size=12001) 2024-11-24T02:43:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:18,894 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/c1b505690a634d7b9281d397188018ff as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c1b505690a634d7b9281d397188018ff 2024-11-24T02:43:18,899 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into c1b505690a634d7b9281d397188018ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:18,899 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:18,899 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416198441; duration=0sec 2024-11-24T02:43:18,899 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:18,899 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:18,908 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/c14d017b52be438794227cdad0f65ab0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/c14d017b52be438794227cdad0f65ab0 2024-11-24T02:43:18,912 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into c14d017b52be438794227cdad0f65ab0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:18,912 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:18,912 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416198442; duration=0sec 2024-11-24T02:43:18,912 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:18,912 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:18,958 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a012f00c66d644a4a87a575b35b4a764 2024-11-24T02:43:18,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/c59458fef91849c4b8aaf7c2e84c16c9 is 50, key is test_row_0/B:col10/1732416197230/Put/seqid=0 2024-11-24T02:43:18,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742091_1267 (size=12001) 2024-11-24T02:43:18,970 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/c59458fef91849c4b8aaf7c2e84c16c9 2024-11-24T02:43:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/721357691930426cbd0603460ed5f032 is 50, key is test_row_0/C:col10/1732416197230/Put/seqid=0 2024-11-24T02:43:18,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742092_1268 (size=12001) 2024-11-24T02:43:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:19,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/721357691930426cbd0603460ed5f032 2024-11-24T02:43:19,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a012f00c66d644a4a87a575b35b4a764 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764 2024-11-24T02:43:19,390 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:43:19,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/c59458fef91849c4b8aaf7c2e84c16c9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9 2024-11-24T02:43:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:19,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:19,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:43:19,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/721357691930426cbd0603460ed5f032 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032 2024-11-24T02:43:19,400 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:43:19,401 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=20.13 KB/20610 for f40cba6b6b9bf91dffe1d7372b246258 in 852ms, sequenceid=116, compaction requested=false 2024-11-24T02:43:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T02:43:19,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T02:43:19,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T02:43:19,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-11-24T02:43:19,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3150 sec 2024-11-24T02:43:19,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:19,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:43:19,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:19,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:19,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:19,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:19,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:19,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:19,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a8c517da88664628bdd2768111e7983d is 50, key is test_row_0/A:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:19,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742093_1269 (size=12001) 2024-11-24T02:43:19,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a8c517da88664628bdd2768111e7983d 2024-11-24T02:43:19,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/46ece3437f5748e2a667cc10d9494f27 is 50, key is test_row_0/B:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:19,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742094_1270 (size=12001) 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416199421, max timestamp=1732416199432Num uniques 859; 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416199421, max timestamp=1732416199432Num uniques 859; 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416199421, max timestamp=1732416199432Num uniques 859; 2024-11-24T02:43:19,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:19,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416259453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416259454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416259456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416259457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416259458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416259569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416259569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416259569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416259569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416259569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416259779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416259779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416259779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416259779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416259779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:19,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/46ece3437f5748e2a667cc10d9494f27 2024-11-24T02:43:19,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8c6027ee9f644a2cbeb343a8f397037c is 50, key is test_row_0/C:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:19,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742095_1271 (size=12001) 2024-11-24T02:43:20,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416260089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416260089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416260089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416260089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416260090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:43:20,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:20,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T02:43:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:20,241 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:20,243 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:20,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:20,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8c6027ee9f644a2cbeb343a8f397037c 2024-11-24T02:43:20,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a8c517da88664628bdd2768111e7983d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d 2024-11-24T02:43:20,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:43:20,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/46ece3437f5748e2a667cc10d9494f27 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27 2024-11-24T02:43:20,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:43:20,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8c6027ee9f644a2cbeb343a8f397037c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c 2024-11-24T02:43:20,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:43:20,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f40cba6b6b9bf91dffe1d7372b246258 in 871ms, sequenceid=128, compaction requested=true 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:20,279 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:20,279 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:20,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:20,280 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:20,280 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:20,280 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c1b505690a634d7b9281d397188018ff, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.4 K 2024-11-24T02:43:20,280 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c349040bad954f909ba0988def9eec79, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.4 K 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b505690a634d7b9281d397188018ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:20,280 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c349040bad954f909ba0988def9eec79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:20,281 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a012f00c66d644a4a87a575b35b4a764, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416197230 2024-11-24T02:43:20,281 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c59458fef91849c4b8aaf7c2e84c16c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416197230 2024-11-24T02:43:20,281 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ece3437f5748e2a667cc10d9494f27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:20,281 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c517da88664628bdd2768111e7983d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:20,299 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:20,299 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:20,300 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/45cc28e2816d4f97bf2e481b24e2a121 is 50, key is test_row_0/A:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:20,300 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/d9db48afc4a047fc92596b39400b4a1f is 50, key is test_row_0/B:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742096_1272 (size=12309) 2024-11-24T02:43:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:20,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742097_1273 (size=12309) 2024-11-24T02:43:20,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:20,395 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/c0afc9b81d504572af53c979f02c7bf9 is 50, key is test_row_0/A:col10/1732416199454/Put/seqid=0 2024-11-24T02:43:20,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742098_1274 (size=12151) 2024-11-24T02:43:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:20,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416260605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416260606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416260605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416260612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416260612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,709 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/45cc28e2816d4f97bf2e481b24e2a121 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/45cc28e2816d4f97bf2e481b24e2a121 2024-11-24T02:43:20,713 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into 45cc28e2816d4f97bf2e481b24e2a121(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:20,713 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:20,713 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416200279; duration=0sec 2024-11-24T02:43:20,713 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:20,713 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:20,714 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:20,714 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:20,715 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:20,715 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:20,715 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/c14d017b52be438794227cdad0f65ab0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.4 K 2024-11-24T02:43:20,715 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14d017b52be438794227cdad0f65ab0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416197172 2024-11-24T02:43:20,715 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 721357691930426cbd0603460ed5f032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416197230 2024-11-24T02:43:20,716 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c6027ee9f644a2cbeb343a8f397037c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416260719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416260719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,724 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,724 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/414fbc1a18ae4cd1b6be31f02f8f3af0 is 50, key is test_row_0/C:col10/1732416199403/Put/seqid=0 2024-11-24T02:43:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416260719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416260719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416260720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742099_1275 (size=12309) 2024-11-24T02:43:20,770 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/d9db48afc4a047fc92596b39400b4a1f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/d9db48afc4a047fc92596b39400b4a1f 2024-11-24T02:43:20,776 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into d9db48afc4a047fc92596b39400b4a1f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:20,776 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:20,776 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416200279; duration=0sec 2024-11-24T02:43:20,776 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:20,776 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:20,809 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/c0afc9b81d504572af53c979f02c7bf9 2024-11-24T02:43:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/1a536f3f75c543b3bd33476841053817 is 50, key is test_row_0/B:col10/1732416199454/Put/seqid=0 2024-11-24T02:43:20,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742100_1276 (size=12151) 2024-11-24T02:43:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416260929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416260929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416260929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416260929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:20,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416260930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:20,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,134 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/414fbc1a18ae4cd1b6be31f02f8f3af0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/414fbc1a18ae4cd1b6be31f02f8f3af0 2024-11-24T02:43:21,139 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 414fbc1a18ae4cd1b6be31f02f8f3af0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:21,139 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:21,139 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416200279; duration=0sec 2024-11-24T02:43:21,140 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:21,140 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:21,222 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/1a536f3f75c543b3bd33476841053817 2024-11-24T02:43:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3d4da6a481d346bbb31a6582004c37d5 is 50, key is test_row_0/C:col10/1732416199454/Put/seqid=0 2024-11-24T02:43:21,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742101_1277 (size=12151) 2024-11-24T02:43:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:21,637 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3d4da6a481d346bbb31a6582004c37d5 2024-11-24T02:43:21,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/c0afc9b81d504572af53c979f02c7bf9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9 2024-11-24T02:43:21,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T02:43:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/1a536f3f75c543b3bd33476841053817 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817 2024-11-24T02:43:21,651 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T02:43:21,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/3d4da6a481d346bbb31a6582004c37d5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5 2024-11-24T02:43:21,656 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5, entries=150, sequenceid=165, filesize=11.9 K 2024-11-24T02:43:21,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for f40cba6b6b9bf91dffe1d7372b246258 in 1262ms, sequenceid=165, compaction requested=false 2024-11-24T02:43:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:21,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T02:43:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T02:43:21,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T02:43:21,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-24T02:43:21,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4210 sec 2024-11-24T02:43:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:21,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:21,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:21,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/fc4e809692ba40558ce0c593b953c548 is 50, key is test_row_0/A:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:21,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742102_1278 (size=14541) 2024-11-24T02:43:21,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416261769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416261772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416261772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416261773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416261775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416261879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416261889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416261889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416261889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416261889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416262089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416262099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416262099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416262099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416262099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/fc4e809692ba40558ce0c593b953c548 2024-11-24T02:43:22,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/8681b2c6110b403e8ef7911b63af1968 is 50, key is test_row_0/B:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:22,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742103_1279 (size=12151) 2024-11-24T02:43:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:43:22,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:22,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T02:43:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:22,391 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:22,391 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:22,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416262399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416262409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416262409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416262409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416262409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:22,518 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:43:22,518 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T02:43:22,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:43:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/8681b2c6110b403e8ef7911b63af1968 2024-11-24T02:43:22,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8e063a824d4b4d59b5225ca3b796240f is 50, key is test_row_0/C:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:22,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742104_1280 (size=12151) 2024-11-24T02:43:22,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:22,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:43:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:22,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:22,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:22,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416262909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416262919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416262919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416262919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416262920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:22,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8e063a824d4b4d59b5225ca3b796240f 2024-11-24T02:43:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/fc4e809692ba40558ce0c593b953c548 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548 2024-11-24T02:43:22,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548, entries=200, sequenceid=180, filesize=14.2 K 2024-11-24T02:43:22,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/8681b2c6110b403e8ef7911b63af1968 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968 2024-11-24T02:43:23,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968, entries=150, sequenceid=180, filesize=11.9 K 2024-11-24T02:43:23,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:43:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:23,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:23,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/8e063a824d4b4d59b5225ca3b796240f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f 2024-11-24T02:43:23,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f, entries=150, sequenceid=180, filesize=11.9 K 2024-11-24T02:43:23,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f40cba6b6b9bf91dffe1d7372b246258 in 1258ms, sequenceid=180, compaction requested=true 2024-11-24T02:43:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:23,010 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:23,010 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:23,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:23,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:23,011 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,011 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/d9db48afc4a047fc92596b39400b4a1f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.8 K 2024-11-24T02:43:23,011 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:23,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d9db48afc4a047fc92596b39400b4a1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:23,011 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:23,011 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,012 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/45cc28e2816d4f97bf2e481b24e2a121, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=38.1 K 2024-11-24T02:43:23,012 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a536f3f75c543b3bd33476841053817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416199454 2024-11-24T02:43:23,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45cc28e2816d4f97bf2e481b24e2a121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:23,012 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8681b2c6110b403e8ef7911b63af1968, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:23,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0afc9b81d504572af53c979f02c7bf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416199454 2024-11-24T02:43:23,013 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc4e809692ba40558ce0c593b953c548, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:23,038 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:23,039 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/75949ea677f94b01af06f73fd7ff86e7 is 50, key is test_row_0/A:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:23,039 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:23,039 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/2ef1693e52bc44d0a7329bf0c0c44676 is 50, key is test_row_0/B:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:23,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742105_1281 (size=12561) 2024-11-24T02:43:23,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742106_1282 (size=12561) 2024-11-24T02:43:23,060 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/2ef1693e52bc44d0a7329bf0c0c44676 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/2ef1693e52bc44d0a7329bf0c0c44676 2024-11-24T02:43:23,067 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into 2ef1693e52bc44d0a7329bf0c0c44676(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:23,067 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:23,067 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416203010; duration=0sec 2024-11-24T02:43:23,067 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:23,067 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:23,067 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:23,068 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:23,068 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:23,068 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,068 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/414fbc1a18ae4cd1b6be31f02f8f3af0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=35.8 K 2024-11-24T02:43:23,069 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 414fbc1a18ae4cd1b6be31f02f8f3af0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416199403 2024-11-24T02:43:23,069 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4da6a481d346bbb31a6582004c37d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416199454 2024-11-24T02:43:23,069 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e063a824d4b4d59b5225ca3b796240f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:23,077 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:23,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/688b9a9fec0349779371f0218fcb4bdd is 50, key is test_row_0/C:col10/1732416200611/Put/seqid=0 2024-11-24T02:43:23,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742107_1283 (size=12561) 2024-11-24T02:43:23,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:43:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:23,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:43:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:23,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/1262d224643842b4aacf8a733fab3201 is 50, key is test_row_0/A:col10/1732416201767/Put/seqid=0 2024-11-24T02:43:23,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742108_1284 (size=12151) 2024-11-24T02:43:23,453 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/75949ea677f94b01af06f73fd7ff86e7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/75949ea677f94b01af06f73fd7ff86e7 2024-11-24T02:43:23,457 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into 75949ea677f94b01af06f73fd7ff86e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:23,457 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:23,457 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416203010; duration=0sec 2024-11-24T02:43:23,457 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:23,457 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:23,503 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/688b9a9fec0349779371f0218fcb4bdd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/688b9a9fec0349779371f0218fcb4bdd 2024-11-24T02:43:23,507 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 688b9a9fec0349779371f0218fcb4bdd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:23,507 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:23,507 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416203010; duration=0sec 2024-11-24T02:43:23,507 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:23,507 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:23,564 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/1262d224643842b4aacf8a733fab3201 2024-11-24T02:43:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/757fe5eb041949719c7e22d30c7c9139 is 50, key is test_row_0/B:col10/1732416201767/Put/seqid=0 2024-11-24T02:43:23,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742109_1285 (size=12151) 2024-11-24T02:43:23,575 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/757fe5eb041949719c7e22d30c7c9139 2024-11-24T02:43:23,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/d181f998b08d4270a11bdc291565db22 is 50, key is test_row_0/C:col10/1732416201767/Put/seqid=0 2024-11-24T02:43:23,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742110_1286 (size=12151) 2024-11-24T02:43:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416263936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416263939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:23,968 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in f40cba6b6b9bf91dffe1d7372b246258 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416263968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416263968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416263968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:23,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:23,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:23,986 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/d181f998b08d4270a11bdc291565db22 2024-11-24T02:43:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/1262d224643842b4aacf8a733fab3201 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201 2024-11-24T02:43:23,996 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T02:43:23,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/757fe5eb041949719c7e22d30c7c9139 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139 2024-11-24T02:43:24,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T02:43:24,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/d181f998b08d4270a11bdc291565db22 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22 2024-11-24T02:43:24,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22, entries=150, sequenceid=201, filesize=11.9 K 2024-11-24T02:43:24,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f40cba6b6b9bf91dffe1d7372b246258 in 852ms, sequenceid=201, compaction requested=false 2024-11-24T02:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T02:43:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T02:43:24,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T02:43:24,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-24T02:43:24,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6220 sec 2024-11-24T02:43:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:24,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/aeff978776c94043a500787271cd17a2 is 50, key is test_row_0/A:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742111_1287 (size=14541) 2024-11-24T02:43:24,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/aeff978776c94043a500787271cd17a2 2024-11-24T02:43:24,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/65e56fd0d9924b538ba06cbd2445e927 is 50, key is test_row_0/B:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742112_1288 (size=12151) 2024-11-24T02:43:24,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416264104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416264105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416264106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416264108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416264219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416264219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416264219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416264219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416264428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416264429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416264429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416264429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/65e56fd0d9924b538ba06cbd2445e927 2024-11-24T02:43:24,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/96d75c676e1c40e59ab7a4ec299bbb2d is 50, key is test_row_0/C:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742113_1289 (size=12151) 2024-11-24T02:43:24,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:43:24,549 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:24,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T02:43:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:24,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:24,552 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:24,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:24,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:43:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416264738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416264738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416264738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416264739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:24,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:43:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:24,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/96d75c676e1c40e59ab7a4ec299bbb2d 2024-11-24T02:43:24,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/aeff978776c94043a500787271cd17a2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2 2024-11-24T02:43:24,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2, entries=200, sequenceid=216, filesize=14.2 K 2024-11-24T02:43:24,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/65e56fd0d9924b538ba06cbd2445e927 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927 2024-11-24T02:43:24,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927, entries=150, sequenceid=216, filesize=11.9 K 2024-11-24T02:43:24,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/96d75c676e1c40e59ab7a4ec299bbb2d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d 2024-11-24T02:43:24,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d, entries=150, sequenceid=216, filesize=11.9 K 2024-11-24T02:43:24,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f40cba6b6b9bf91dffe1d7372b246258 in 845ms, sequenceid=216, compaction requested=true 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:24,926 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:24,926 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:24,927 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:24,927 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:24,927 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:24,927 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:24,927 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,927 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,927 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/75949ea677f94b01af06f73fd7ff86e7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=38.3 K 2024-11-24T02:43:24,927 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/2ef1693e52bc44d0a7329bf0c0c44676, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.0 K 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75949ea677f94b01af06f73fd7ff86e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef1693e52bc44d0a7329bf0c0c44676, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1262d224643842b4aacf8a733fab3201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416201767 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 757fe5eb041949719c7e22d30c7c9139, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416201767 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeff978776c94043a500787271cd17a2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:24,928 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 65e56fd0d9924b538ba06cbd2445e927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:24,935 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:24,935 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:24,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/2d337c0a8cac4b2b98681bab382af454 is 50, key is test_row_0/A:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,936 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/a0135945fb32493db96523ef85792b61 is 50, key is test_row_0/B:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742114_1290 (size=12663) 2024-11-24T02:43:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742115_1291 (size=12663) 2024-11-24T02:43:24,949 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/a0135945fb32493db96523ef85792b61 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a0135945fb32493db96523ef85792b61 2024-11-24T02:43:24,954 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into a0135945fb32493db96523ef85792b61(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:24,954 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:24,954 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416204926; duration=0sec 2024-11-24T02:43:24,954 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:24,954 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:24,954 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:24,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:24,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:24,956 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:24,956 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/688b9a9fec0349779371f0218fcb4bdd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.0 K 2024-11-24T02:43:24,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 688b9a9fec0349779371f0218fcb4bdd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732416200604 2024-11-24T02:43:24,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d181f998b08d4270a11bdc291565db22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732416201767 2024-11-24T02:43:24,957 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 96d75c676e1c40e59ab7a4ec299bbb2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:24,963 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:24,963 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/058b329c79e340d1ae7bfc1d3967e014 is 50, key is test_row_0/C:col10/1732416203932/Put/seqid=0 2024-11-24T02:43:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742116_1292 (size=12663) 2024-11-24T02:43:25,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:43:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:25,009 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:43:25,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:25,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/07e559871b39496aa35af05591e14a50 is 50, key is test_row_0/A:col10/1732416204105/Put/seqid=0 2024-11-24T02:43:25,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742117_1293 (size=12151) 2024-11-24T02:43:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:25,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416265258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416265258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416265260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416265262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416265369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416265369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416265369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416265369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,397 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/058b329c79e340d1ae7bfc1d3967e014 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/058b329c79e340d1ae7bfc1d3967e014 2024-11-24T02:43:25,397 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/2d337c0a8cac4b2b98681bab382af454 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/2d337c0a8cac4b2b98681bab382af454 2024-11-24T02:43:25,401 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 058b329c79e340d1ae7bfc1d3967e014(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:25,401 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:25,401 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416204926; duration=0sec 2024-11-24T02:43:25,401 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:25,401 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:25,405 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into 2d337c0a8cac4b2b98681bab382af454(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:25,405 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:25,405 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416204926; duration=0sec 2024-11-24T02:43:25,405 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:25,406 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:25,419 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/07e559871b39496aa35af05591e14a50 2024-11-24T02:43:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f2d61becc81143f28d539e4ae7da35e8 is 50, key is test_row_0/B:col10/1732416204105/Put/seqid=0 2024-11-24T02:43:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742118_1294 (size=12151) 2024-11-24T02:43:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416265578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416265579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416265579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416265579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:25,830 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f2d61becc81143f28d539e4ae7da35e8 2024-11-24T02:43:25,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/93cd71c79f2f4a51b45730fa6e42efa2 is 50, key is test_row_0/C:col10/1732416204105/Put/seqid=0 2024-11-24T02:43:25,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742119_1295 (size=12151) 2024-11-24T02:43:25,842 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/93cd71c79f2f4a51b45730fa6e42efa2 2024-11-24T02:43:25,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/07e559871b39496aa35af05591e14a50 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50 2024-11-24T02:43:25,850 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50, entries=150, sequenceid=236, filesize=11.9 K 2024-11-24T02:43:25,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f2d61becc81143f28d539e4ae7da35e8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8 2024-11-24T02:43:25,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8, entries=150, sequenceid=236, filesize=11.9 K 2024-11-24T02:43:25,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/93cd71c79f2f4a51b45730fa6e42efa2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2 2024-11-24T02:43:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2, entries=150, sequenceid=236, filesize=11.9 K 2024-11-24T02:43:25,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f40cba6b6b9bf91dffe1d7372b246258 in 852ms, sequenceid=236, compaction requested=false 2024-11-24T02:43:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:25,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T02:43:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T02:43:25,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T02:43:25,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-11-24T02:43:25,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3160 sec 2024-11-24T02:43:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:25,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:25,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:25,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/73a5c09ef6e14168abeb0312b2ca2433 is 50, key is test_row_0/A:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742120_1296 (size=12151) 2024-11-24T02:43:25,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/73a5c09ef6e14168abeb0312b2ca2433 2024-11-24T02:43:25,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416265906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416265908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416265909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416265912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:25,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/90f819390d1d48eabf91b7b267c2b537 is 50, key is test_row_0/B:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:25,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742121_1297 (size=12151) 2024-11-24T02:43:25,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416265989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:25,991 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in f40cba6b6b9bf91dffe1d7372b246258 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416266019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416266019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416266019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416266019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416266229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416266229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416266229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416266230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/90f819390d1d48eabf91b7b267c2b537 2024-11-24T02:43:26,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/81394753c34148f990413bd429d457cc is 50, key is test_row_0/C:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:26,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742122_1298 (size=12151) 2024-11-24T02:43:26,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/81394753c34148f990413bd429d457cc 2024-11-24T02:43:26,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/73a5c09ef6e14168abeb0312b2ca2433 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433 2024-11-24T02:43:26,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T02:43:26,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/90f819390d1d48eabf91b7b267c2b537 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537 2024-11-24T02:43:26,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T02:43:26,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/81394753c34148f990413bd429d457cc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc 2024-11-24T02:43:26,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T02:43:26,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f40cba6b6b9bf91dffe1d7372b246258 in 459ms, sequenceid=253, compaction requested=true 2024-11-24T02:43:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:26,350 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:26,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:26,351 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:26,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:26,351 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:26,351 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:26,351 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:26,351 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:26,351 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:26,351 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:26,351 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a0135945fb32493db96523ef85792b61, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.1 K 2024-11-24T02:43:26,351 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/2d337c0a8cac4b2b98681bab382af454, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.1 K 2024-11-24T02:43:26,352 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d337c0a8cac4b2b98681bab382af454, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:26,352 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a0135945fb32493db96523ef85792b61, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:26,353 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d61becc81143f28d539e4ae7da35e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732416204101 2024-11-24T02:43:26,353 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e559871b39496aa35af05591e14a50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732416204101 2024-11-24T02:43:26,353 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f819390d1d48eabf91b7b267c2b537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:26,353 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a5c09ef6e14168abeb0312b2ca2433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:26,362 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:26,362 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:26,362 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b01123df8f8047138e4ed74908750573 is 50, key is test_row_0/A:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:26,364 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/3e848b82010c43dcbdb1b0b9b31d299d is 50, key is test_row_0/B:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:26,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742123_1299 (size=12765) 2024-11-24T02:43:26,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742124_1300 (size=12765) 2024-11-24T02:43:26,385 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b01123df8f8047138e4ed74908750573 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b01123df8f8047138e4ed74908750573 2024-11-24T02:43:26,389 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/3e848b82010c43dcbdb1b0b9b31d299d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/3e848b82010c43dcbdb1b0b9b31d299d 2024-11-24T02:43:26,392 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into b01123df8f8047138e4ed74908750573(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:26,392 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:26,392 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416206350; duration=0sec 2024-11-24T02:43:26,392 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:26,392 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:26,392 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:26,393 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:26,394 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:26,394 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:26,394 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/058b329c79e340d1ae7bfc1d3967e014, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.1 K 2024-11-24T02:43:26,394 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 058b329c79e340d1ae7bfc1d3967e014, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732416203932 2024-11-24T02:43:26,395 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93cd71c79f2f4a51b45730fa6e42efa2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732416204101 2024-11-24T02:43:26,395 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81394753c34148f990413bd429d457cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:26,395 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into 3e848b82010c43dcbdb1b0b9b31d299d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:26,395 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:26,395 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416206350; duration=0sec 2024-11-24T02:43:26,395 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:26,395 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:26,402 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:26,403 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/872c114d53d9475b8d4426ee113f2346 is 50, key is test_row_0/C:col10/1732416205890/Put/seqid=0 2024-11-24T02:43:26,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742125_1301 (size=12765) 2024-11-24T02:43:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:26,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:26,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:26,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/d56442a747654d21ab0181720f3ac66c is 50, key is test_row_0/A:col10/1732416206542/Put/seqid=0 2024-11-24T02:43:26,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416266553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416266555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416266555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416266556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742126_1302 (size=12301) 2024-11-24T02:43:26,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416266659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416266669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416266669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416266669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:43:26,699 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:26,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T02:43:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:26,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:26,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:26,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:26,810 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/872c114d53d9475b8d4426ee113f2346 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/872c114d53d9475b8d4426ee113f2346 2024-11-24T02:43:26,815 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 872c114d53d9475b8d4426ee113f2346(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:26,815 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:26,815 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416206351; duration=0sec 2024-11-24T02:43:26,815 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:26,815 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:26,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:43:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:26,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416266869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416266878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416266878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416266879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:26,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:26,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/d56442a747654d21ab0181720f3ac66c 2024-11-24T02:43:26,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/39eb7afe6d444787bd27c833cc39d0e7 is 50, key is test_row_0/B:col10/1732416206542/Put/seqid=0 2024-11-24T02:43:26,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742127_1303 (size=12301) 2024-11-24T02:43:26,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/39eb7afe6d444787bd27c833cc39d0e7 2024-11-24T02:43:26,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/1eac0aadfc59415a98933f159ddc7c87 is 50, key is test_row_0/C:col10/1732416206542/Put/seqid=0 2024-11-24T02:43:26,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742128_1304 (size=12301) 2024-11-24T02:43:27,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:43:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:27,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416267179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416267188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416267188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416267189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:43:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:27,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/1eac0aadfc59415a98933f159ddc7c87 2024-11-24T02:43:27,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/d56442a747654d21ab0181720f3ac66c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c 2024-11-24T02:43:27,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:43:27,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/39eb7afe6d444787bd27c833cc39d0e7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7 2024-11-24T02:43:27,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:43:27,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/1eac0aadfc59415a98933f159ddc7c87 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87 2024-11-24T02:43:27,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:43:27,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f40cba6b6b9bf91dffe1d7372b246258 in 863ms, sequenceid=274, compaction requested=false 2024-11-24T02:43:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:27,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:43:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:27,464 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:43:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:27,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b2d0bcf765eb439482d2af806d6ef0db is 50, key is test_row_0/A:col10/1732416206554/Put/seqid=0 2024-11-24T02:43:27,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742129_1305 (size=12301) 2024-11-24T02:43:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:27,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:27,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416267715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416267715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416267716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416267721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416267829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416267829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416267829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416267829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:27,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b2d0bcf765eb439482d2af806d6ef0db 2024-11-24T02:43:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b566da5cef17407785819ce43051e5b1 is 50, key is test_row_0/B:col10/1732416206554/Put/seqid=0 2024-11-24T02:43:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742130_1306 (size=12301) 2024-11-24T02:43:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416268039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416268039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416268039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416268039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b566da5cef17407785819ce43051e5b1 2024-11-24T02:43:28,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/14f9c360b088495594fa2334235d61b7 is 50, key is test_row_0/C:col10/1732416206554/Put/seqid=0 2024-11-24T02:43:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742131_1307 (size=12301) 2024-11-24T02:43:28,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416268349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416268349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416268349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416268350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/14f9c360b088495594fa2334235d61b7 2024-11-24T02:43:28,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b2d0bcf765eb439482d2af806d6ef0db as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db 2024-11-24T02:43:28,709 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:43:28,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/b566da5cef17407785819ce43051e5b1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1 2024-11-24T02:43:28,713 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:43:28,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/14f9c360b088495594fa2334235d61b7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7 2024-11-24T02:43:28,717 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:43:28,718 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f40cba6b6b9bf91dffe1d7372b246258 in 1254ms, sequenceid=288, compaction requested=true 2024-11-24T02:43:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T02:43:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T02:43:28,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T02:43:28,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-24T02:43:28,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0230 sec 2024-11-24T02:43:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:43:28,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:28,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T02:43:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:28,851 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:28,852 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:28,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:28,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:28,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:28,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ed46a118328349ca85c6765354bfc0b5 is 50, key is test_row_0/A:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:28,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742132_1308 (size=12301) 2024-11-24T02:43:28,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ed46a118328349ca85c6765354bfc0b5 2024-11-24T02:43:28,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/7a5b630e5b0c4687acd30d04ceaa8d6b is 50, key is test_row_0/B:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416268871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416268872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416268874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742133_1309 (size=12301) 2024-11-24T02:43:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416268877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416268989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416268989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416268989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416268990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416269199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/7a5b630e5b0c4687acd30d04ceaa8d6b 2024-11-24T02:43:29,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/781bb64b8dda4ff0ab4ff66f37174bad is 50, key is test_row_0/C:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:29,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742134_1310 (size=12301) 2024-11-24T02:43:29,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:29,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416269509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416269509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416269509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416269509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:29,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:29,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/781bb64b8dda4ff0ab4ff66f37174bad 2024-11-24T02:43:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/ed46a118328349ca85c6765354bfc0b5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5 2024-11-24T02:43:29,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T02:43:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/7a5b630e5b0c4687acd30d04ceaa8d6b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b 2024-11-24T02:43:29,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T02:43:29,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/781bb64b8dda4ff0ab4ff66f37174bad as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad 2024-11-24T02:43:29,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T02:43:29,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f40cba6b6b9bf91dffe1d7372b246258 in 845ms, sequenceid=308, compaction requested=true 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:29,707 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:43:29,707 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:43:29,708 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:43:29,708 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:43:29,708 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:29,708 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:29,708 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,708 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,709 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/3e848b82010c43dcbdb1b0b9b31d299d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=48.5 K 2024-11-24T02:43:29,709 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b01123df8f8047138e4ed74908750573, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=48.5 K 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01123df8f8047138e4ed74908750573, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e848b82010c43dcbdb1b0b9b31d299d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56442a747654d21ab0181720f3ac66c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416205907 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 39eb7afe6d444787bd27c833cc39d0e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416205907 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d0bcf765eb439482d2af806d6ef0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416206553 2024-11-24T02:43:29,709 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b566da5cef17407785819ce43051e5b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416206553 2024-11-24T02:43:29,710 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed46a118328349ca85c6765354bfc0b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:29,710 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5b630e5b0c4687acd30d04ceaa8d6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:29,718 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:29,718 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:29,718 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/102e5d241539471e88a25403a07cfc59 is 50, key is test_row_0/B:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:29,719 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b6131f27b2764998b10b99d32e9c0707 is 50, key is test_row_0/A:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:29,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742135_1311 (size=13051) 2024-11-24T02:43:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742136_1312 (size=13051) 2024-11-24T02:43:29,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:43:29,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:29,768 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a91140b800f641b793b7493fffae755f is 50, key is test_row_0/A:col10/1732416208870/Put/seqid=0 2024-11-24T02:43:29,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742137_1313 (size=12301) 2024-11-24T02:43:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:30,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:30,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416270033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416270033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732416210030, max timestamp=1732416210047Num uniques 859; 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732416210030, max timestamp=1732416210047Num uniques 859; 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732416210030, max timestamp=1732416210047Num uniques 859; 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T02:43:30,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:43:30,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416270097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416270098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416270098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,130 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b6131f27b2764998b10b99d32e9c0707 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b6131f27b2764998b10b99d32e9c0707 2024-11-24T02:43:30,134 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/102e5d241539471e88a25403a07cfc59 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/102e5d241539471e88a25403a07cfc59 2024-11-24T02:43:30,135 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into b6131f27b2764998b10b99d32e9c0707(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:30,135 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:30,135 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=12, startTime=1732416209707; duration=0sec 2024-11-24T02:43:30,135 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:30,135 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:30,135 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:43:30,141 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:43:30,141 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:30,141 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:30,141 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/872c114d53d9475b8d4426ee113f2346, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=48.5 K 2024-11-24T02:43:30,142 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872c114d53d9475b8d4426ee113f2346, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732416205257 2024-11-24T02:43:30,142 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eac0aadfc59415a98933f159ddc7c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416205907 2024-11-24T02:43:30,142 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f9c360b088495594fa2334235d61b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416206553 2024-11-24T02:43:30,143 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781bb64b8dda4ff0ab4ff66f37174bad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:30,145 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into 102e5d241539471e88a25403a07cfc59(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:30,145 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:30,145 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=12, startTime=1732416209707; duration=0sec 2024-11-24T02:43:30,145 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:30,145 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:30,151 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:30,151 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/85cc6596c0f342bb8e6b04907a979eac is 50, key is test_row_0/C:col10/1732416207719/Put/seqid=0 2024-11-24T02:43:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742138_1314 (size=13051) 2024-11-24T02:43:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416270169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416270169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a91140b800f641b793b7493fffae755f 2024-11-24T02:43:30,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/43cb0f36f02046389a65ff07184a931f is 50, key is test_row_0/B:col10/1732416208870/Put/seqid=0 2024-11-24T02:43:30,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742139_1315 (size=12301) 2024-11-24T02:43:30,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416270209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416270209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416270209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416270379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416270379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416270419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416270419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416270419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,561 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/85cc6596c0f342bb8e6b04907a979eac as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/85cc6596c0f342bb8e6b04907a979eac 2024-11-24T02:43:30,565 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into 85cc6596c0f342bb8e6b04907a979eac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:30,565 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:30,566 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=12, startTime=1732416209707; duration=0sec 2024-11-24T02:43:30,566 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:30,566 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:30,591 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/43cb0f36f02046389a65ff07184a931f 2024-11-24T02:43:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/21d1c09d75d848d19c7ce79f988a0bd4 is 50, key is test_row_0/C:col10/1732416208870/Put/seqid=0 2024-11-24T02:43:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742140_1316 (size=12301) 2024-11-24T02:43:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416270689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416270689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416270729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416270729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416270729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:31,001 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/21d1c09d75d848d19c7ce79f988a0bd4 2024-11-24T02:43:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/a91140b800f641b793b7493fffae755f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f 2024-11-24T02:43:31,008 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T02:43:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/43cb0f36f02046389a65ff07184a931f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f 2024-11-24T02:43:31,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T02:43:31,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/21d1c09d75d848d19c7ce79f988a0bd4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4 2024-11-24T02:43:31,016 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T02:43:31,017 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=122.60 KB/125546 for f40cba6b6b9bf91dffe1d7372b246258 in 1249ms, sequenceid=320, compaction requested=false 2024-11-24T02:43:31,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:31,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:31,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T02:43:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T02:43:31,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T02:43:31,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1660 sec 2024-11-24T02:43:31,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1700 sec 2024-11-24T02:43:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:31,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=129.31 KB heapSize=364.54 KB 2024-11-24T02:43:31,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:31,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:31,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:31,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:31,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:31,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:31,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/92d2fa60588444e2ad05021b620779fb is 50, key is test_row_0/A:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:31,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742141_1317 (size=17181) 2024-11-24T02:43:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416271222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416271224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416271238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416271238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416271239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:43:31,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416271328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416271339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/92d2fa60588444e2ad05021b620779fb 2024-11-24T02:43:31,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416271642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416271642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/a131c1af37324fc6926443be34d497c4 is 50, key is test_row_0/B:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:31,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742142_1318 (size=12301) 2024-11-24T02:43:31,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416271949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:31,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:31,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416271949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:31,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/a131c1af37324fc6926443be34d497c4 2024-11-24T02:43:32,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/668f170a3d434e57b5894f426debfe2e is 50, key is test_row_0/C:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:32,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742143_1319 (size=12301) 2024-11-24T02:43:32,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45800 deadline: 1732416272248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:32,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45884 deadline: 1732416272248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:32,251 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in f40cba6b6b9bf91dffe1d7372b246258 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45798 deadline: 1732416272248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:32,251 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in f40cba6b6b9bf91dffe1d7372b246258 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,252 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in f40cba6b6b9bf91dffe1d7372b246258 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416272459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416272459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:32,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.10 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/668f170a3d434e57b5894f426debfe2e 2024-11-24T02:43:32,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/92d2fa60588444e2ad05021b620779fb as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb 2024-11-24T02:43:32,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb, entries=250, sequenceid=359, filesize=16.8 K 2024-11-24T02:43:32,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/a131c1af37324fc6926443be34d497c4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4 2024-11-24T02:43:32,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:43:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/668f170a3d434e57b5894f426debfe2e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e 2024-11-24T02:43:32,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:43:32,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139286, heapSize ~384.41 KB/393640, currentSize=46.96 KB/48090 for f40cba6b6b9bf91dffe1d7372b246258 in 1296ms, sequenceid=359, compaction requested=true 2024-11-24T02:43:32,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:32,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:32,499 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:32,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:32,499 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:32,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:32,502 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:32,502 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/B is initiating minor compaction (all files) 2024-11-24T02:43:32,502 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/B in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:32,502 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/102e5d241539471e88a25403a07cfc59, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.8 K 2024-11-24T02:43:32,502 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:32,502 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/A is initiating minor compaction (all files) 2024-11-24T02:43:32,502 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/A in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:32,503 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b6131f27b2764998b10b99d32e9c0707, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=41.5 K 2024-11-24T02:43:32,503 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 102e5d241539471e88a25403a07cfc59, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:32,503 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6131f27b2764998b10b99d32e9c0707, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:32,503 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a91140b800f641b793b7493fffae755f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732416208870 2024-11-24T02:43:32,503 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 43cb0f36f02046389a65ff07184a931f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732416208870 2024-11-24T02:43:32,504 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d2fa60588444e2ad05021b620779fb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416210096 2024-11-24T02:43:32,504 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a131c1af37324fc6926443be34d497c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416210097 2024-11-24T02:43:32,527 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:32,530 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/49b698dfe65642179bee8a79167d41d6 is 50, key is test_row_0/B:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:32,537 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:32,538 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b7c731db2ef34fa88c861a84e9a11a57 is 50, key is test_row_0/A:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:32,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742144_1320 (size=13153) 2024-11-24T02:43:32,588 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/49b698dfe65642179bee8a79167d41d6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/49b698dfe65642179bee8a79167d41d6 2024-11-24T02:43:32,597 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/B of f40cba6b6b9bf91dffe1d7372b246258 into 49b698dfe65642179bee8a79167d41d6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:32,597 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:32,597 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/B, priority=13, startTime=1732416212499; duration=0sec 2024-11-24T02:43:32,597 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:32,597 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:32,597 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:32,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742145_1321 (size=13153) 2024-11-24T02:43:32,599 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:32,599 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): f40cba6b6b9bf91dffe1d7372b246258/C is initiating minor compaction (all files) 2024-11-24T02:43:32,599 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f40cba6b6b9bf91dffe1d7372b246258/C in TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:32,599 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/85cc6596c0f342bb8e6b04907a979eac, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp, totalSize=36.8 K 2024-11-24T02:43:32,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 85cc6596c0f342bb8e6b04907a979eac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732416207714 2024-11-24T02:43:32,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d1c09d75d848d19c7ce79f988a0bd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732416208870 2024-11-24T02:43:32,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 668f170a3d434e57b5894f426debfe2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416210097 2024-11-24T02:43:32,610 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f40cba6b6b9bf91dffe1d7372b246258#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:32,612 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/daa7f79edb414b978b39e7e41be4d4de is 50, key is test_row_0/C:col10/1732416211201/Put/seqid=0 2024-11-24T02:43:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742146_1322 (size=13153) 2024-11-24T02:43:32,648 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/daa7f79edb414b978b39e7e41be4d4de as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/daa7f79edb414b978b39e7e41be4d4de 2024-11-24T02:43:32,655 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/C of f40cba6b6b9bf91dffe1d7372b246258 into daa7f79edb414b978b39e7e41be4d4de(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:32,655 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:32,656 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/C, priority=13, startTime=1732416212500; duration=0sec 2024-11-24T02:43:32,656 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:32,656 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:33,003 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b7c731db2ef34fa88c861a84e9a11a57 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b7c731db2ef34fa88c861a84e9a11a57 2024-11-24T02:43:33,009 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f40cba6b6b9bf91dffe1d7372b246258/A of f40cba6b6b9bf91dffe1d7372b246258 into b7c731db2ef34fa88c861a84e9a11a57(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:33,009 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:33,009 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., storeName=f40cba6b6b9bf91dffe1d7372b246258/A, priority=13, startTime=1732416212499; duration=0sec 2024-11-24T02:43:33,009 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:33,009 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:43:33,019 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:33,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T02:43:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:43:33,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:33,022 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:33,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:43:33,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:33,174 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:33,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/f51faf8f2c7d4123848fb661f9970f95 is 50, key is test_row_0/A:col10/1732416211223/Put/seqid=0 2024-11-24T02:43:33,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742147_1323 (size=12301) 2024-11-24T02:43:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:43:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:33,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. as already flushing 2024-11-24T02:43:33,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416213488, max timestamp=1732416213504Num uniques 859; 2024-11-24T02:43:33,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:33,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416213488, max timestamp=1732416213504Num uniques 859; 2024-11-24T02:43:33,506 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416213488, max timestamp=1732416213504Num uniques 859; 2024-11-24T02:43:33,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:33,507 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:43:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416273543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416273544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:33,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/f51faf8f2c7d4123848fb661f9970f95 2024-11-24T02:43:33,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f0eda888748049998b99ab518f4775c9 is 50, key is test_row_0/B:col10/1732416211223/Put/seqid=0 2024-11-24T02:43:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742148_1324 (size=12301) 2024-11-24T02:43:33,597 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f0eda888748049998b99ab518f4775c9 2024-11-24T02:43:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/03828ce8a8c94021b9335b205a9d8030 is 50, key is test_row_0/C:col10/1732416211223/Put/seqid=0 2024-11-24T02:43:33,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742149_1325 (size=12301) 2024-11-24T02:43:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:43:33,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416273659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416273659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:33,787 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-24T02:43:33,787 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:33,787 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,788 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:33,790 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-24T02:43:33,790 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:33,790 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,790 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:33,790 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-24T02:43:33,790 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:33,790 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,790 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:33,792 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-24T02:43:33,792 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:33,792 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,792 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:33,794 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-24T02:43:33,794 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:33,794 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,794 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:33,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45814 deadline: 1732416273869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45856 deadline: 1732416273870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:43:34,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/03828ce8a8c94021b9335b205a9d8030 2024-11-24T02:43:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/f51faf8f2c7d4123848fb661f9970f95 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/f51faf8f2c7d4123848fb661f9970f95 2024-11-24T02:43:34,013 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/f51faf8f2c7d4123848fb661f9970f95, entries=150, sequenceid=372, filesize=12.0 K 2024-11-24T02:43:34,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/f0eda888748049998b99ab518f4775c9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f0eda888748049998b99ab518f4775c9 2024-11-24T02:43:34,017 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f0eda888748049998b99ab518f4775c9, entries=150, sequenceid=372, filesize=12.0 K 2024-11-24T02:43:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/03828ce8a8c94021b9335b205a9d8030 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/03828ce8a8c94021b9335b205a9d8030 2024-11-24T02:43:34,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/03828ce8a8c94021b9335b205a9d8030, entries=150, sequenceid=372, filesize=12.0 K 2024-11-24T02:43:34,021 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f40cba6b6b9bf91dffe1d7372b246258 in 847ms, sequenceid=372, compaction requested=false 2024-11-24T02:43:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:34,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T02:43:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T02:43:34,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T02:43:34,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-11-24T02:43:34,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0060 sec 2024-11-24T02:43:34,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:43:34,160 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:34,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=147.60 KB heapSize=425.53 KB 2024-11-24T02:43:34,182 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-24T02:43:34,182 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-24T02:43:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:34,182 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:34,182 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:43:34,182 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,182 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:43:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:34,183 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,183 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:43:34,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b389c941b05f4d7ba7020db36ef59379 is 50, key is test_row_0/A:col10/1732416214180/Put/seqid=0 2024-11-24T02:43:34,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742150_1326 (size=12301) 2024-11-24T02:43:34,263 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-24T02:43:34,263 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:34,263 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,263 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,271 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-24T02:43:34,271 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-24T02:43:34,271 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:34,271 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:43:34,271 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,271 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,271 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,271 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3255 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3429 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3211 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3262 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3434 2024-11-24T02:43:34,271 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:43:34,271 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:43:34,272 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:43:34,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:34,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:43:34,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:43:34,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:34,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:34,276 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416214276"}]},"ts":"1732416214276"} 2024-11-24T02:43:34,277 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:43:34,277 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:43:34,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:43:34,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, UNASSIGN}] 2024-11-24T02:43:34,281 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, UNASSIGN 2024-11-24T02:43:34,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f40cba6b6b9bf91dffe1d7372b246258, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:34,282 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, UNASSIGN because future has completed 2024-11-24T02:43:34,283 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:43:34,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:34,435 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:34,435 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:43:34,435 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing f40cba6b6b9bf91dffe1d7372b246258, disabling compactions & flushes 2024-11-24T02:43:34,435 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:34,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b389c941b05f4d7ba7020db36ef59379 2024-11-24T02:43:34,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/6fd609989a164ccda04665102ad50bcd is 50, key is test_row_0/B:col10/1732416214180/Put/seqid=0 2024-11-24T02:43:34,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742151_1327 (size=12301) 2024-11-24T02:43:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:35,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/6fd609989a164ccda04665102ad50bcd 2024-11-24T02:43:35,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/51a5a45b17e44323a63cbbed9b2a155f is 50, key is test_row_0/C:col10/1732416214180/Put/seqid=0 2024-11-24T02:43:35,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742152_1328 (size=12301) 2024-11-24T02:43:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:35,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/51a5a45b17e44323a63cbbed9b2a155f 2024-11-24T02:43:35,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b389c941b05f4d7ba7020db36ef59379 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b389c941b05f4d7ba7020db36ef59379 2024-11-24T02:43:35,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b389c941b05f4d7ba7020db36ef59379, entries=150, sequenceid=411, filesize=12.0 K 2024-11-24T02:43:35,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/6fd609989a164ccda04665102ad50bcd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/6fd609989a164ccda04665102ad50bcd 2024-11-24T02:43:35,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/6fd609989a164ccda04665102ad50bcd, entries=150, sequenceid=411, filesize=12.0 K 2024-11-24T02:43:35,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/51a5a45b17e44323a63cbbed9b2a155f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/51a5a45b17e44323a63cbbed9b2a155f 2024-11-24T02:43:35,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/51a5a45b17e44323a63cbbed9b2a155f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-24T02:43:35,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=20.13 KB/20610 for f40cba6b6b9bf91dffe1d7372b246258 in 1332ms, sequenceid=411, compaction requested=true 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f40cba6b6b9bf91dffe1d7372b246258: 2024-11-24T02:43:35,514 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. because compaction request was cancelled 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:A 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. after waiting 0 ms 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:B, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. because compaction request was cancelled 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:B 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f40cba6b6b9bf91dffe1d7372b246258:C, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:35,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. because compaction request was cancelled 2024-11-24T02:43:35,514 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f40cba6b6b9bf91dffe1d7372b246258:C 2024-11-24T02:43:35,514 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing f40cba6b6b9bf91dffe1d7372b246258 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=A 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=B 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:35,514 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f40cba6b6b9bf91dffe1d7372b246258, store=C 2024-11-24T02:43:35,515 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:35,517 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b345dbfa39594beaa320ac0cc884fed3 is 50, key is test_row_0/A:col10/1732416214261/Put/seqid=0 2024-11-24T02:43:35,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742153_1329 (size=12301) 2024-11-24T02:43:35,921 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b345dbfa39594beaa320ac0cc884fed3 2024-11-24T02:43:35,931 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/ef3a6f38b3e547b8b9c30acf25f11fb0 is 50, key is test_row_0/B:col10/1732416214261/Put/seqid=0 2024-11-24T02:43:35,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742154_1330 (size=12301) 2024-11-24T02:43:36,336 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/ef3a6f38b3e547b8b9c30acf25f11fb0 2024-11-24T02:43:36,348 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/7f15fa27adea43fe971c939a2345efcf is 50, key is test_row_0/C:col10/1732416214261/Put/seqid=0 2024-11-24T02:43:36,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742155_1331 (size=12301) 2024-11-24T02:43:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:36,753 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/7f15fa27adea43fe971c939a2345efcf 2024-11-24T02:43:36,763 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/A/b345dbfa39594beaa320ac0cc884fed3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b345dbfa39594beaa320ac0cc884fed3 2024-11-24T02:43:36,769 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b345dbfa39594beaa320ac0cc884fed3, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T02:43:36,769 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/B/ef3a6f38b3e547b8b9c30acf25f11fb0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/ef3a6f38b3e547b8b9c30acf25f11fb0 2024-11-24T02:43:36,773 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/ef3a6f38b3e547b8b9c30acf25f11fb0, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T02:43:36,774 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/.tmp/C/7f15fa27adea43fe971c939a2345efcf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/7f15fa27adea43fe971c939a2345efcf 2024-11-24T02:43:36,778 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/7f15fa27adea43fe971c939a2345efcf, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T02:43:36,779 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f40cba6b6b9bf91dffe1d7372b246258 in 1265ms, sequenceid=417, compaction requested=true 2024-11-24T02:43:36,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/3fd21a7d34fe49708de85bcc9f8914fe, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c1b505690a634d7b9281d397188018ff, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/45cc28e2816d4f97bf2e481b24e2a121, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/75949ea677f94b01af06f73fd7ff86e7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/2d337c0a8cac4b2b98681bab382af454, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b01123df8f8047138e4ed74908750573, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b6131f27b2764998b10b99d32e9c0707, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb] to archive 2024-11-24T02:43:36,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:36,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a1c4876518094b96aa829cc77f20f4c1 2024-11-24T02:43:36,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ac3fc9c6fea44091b10b7f054a6adc2d 2024-11-24T02:43:36,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/48888cf52db84efb86a764d7b81c36a0 2024-11-24T02:43:36,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/3fd21a7d34fe49708de85bcc9f8914fe to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/3fd21a7d34fe49708de85bcc9f8914fe 2024-11-24T02:43:36,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/acec304c5c50433da42228ad7602b903 2024-11-24T02:43:36,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c1b505690a634d7b9281d397188018ff to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c1b505690a634d7b9281d397188018ff 2024-11-24T02:43:36,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/de7cd734cb794a6e836e279ab2fbc0e9 2024-11-24T02:43:36,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a012f00c66d644a4a87a575b35b4a764 2024-11-24T02:43:36,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/45cc28e2816d4f97bf2e481b24e2a121 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/45cc28e2816d4f97bf2e481b24e2a121 2024-11-24T02:43:36,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a8c517da88664628bdd2768111e7983d 2024-11-24T02:43:36,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/c0afc9b81d504572af53c979f02c7bf9 2024-11-24T02:43:36,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/fc4e809692ba40558ce0c593b953c548 2024-11-24T02:43:36,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/75949ea677f94b01af06f73fd7ff86e7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/75949ea677f94b01af06f73fd7ff86e7 2024-11-24T02:43:36,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/1262d224643842b4aacf8a733fab3201 2024-11-24T02:43:36,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/aeff978776c94043a500787271cd17a2 2024-11-24T02:43:36,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/2d337c0a8cac4b2b98681bab382af454 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/2d337c0a8cac4b2b98681bab382af454 2024-11-24T02:43:36,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/07e559871b39496aa35af05591e14a50 2024-11-24T02:43:36,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b01123df8f8047138e4ed74908750573 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b01123df8f8047138e4ed74908750573 2024-11-24T02:43:36,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/73a5c09ef6e14168abeb0312b2ca2433 2024-11-24T02:43:36,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/d56442a747654d21ab0181720f3ac66c 2024-11-24T02:43:36,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b2d0bcf765eb439482d2af806d6ef0db 2024-11-24T02:43:36,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b6131f27b2764998b10b99d32e9c0707 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b6131f27b2764998b10b99d32e9c0707 2024-11-24T02:43:36,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/ed46a118328349ca85c6765354bfc0b5 2024-11-24T02:43:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/a91140b800f641b793b7493fffae755f 2024-11-24T02:43:36,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/92d2fa60588444e2ad05021b620779fb 2024-11-24T02:43:36,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1ee41f9c639345dcbb20e8c1f17cdfbf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c349040bad954f909ba0988def9eec79, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/d9db48afc4a047fc92596b39400b4a1f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/2ef1693e52bc44d0a7329bf0c0c44676, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a0135945fb32493db96523ef85792b61, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/3e848b82010c43dcbdb1b0b9b31d299d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/102e5d241539471e88a25403a07cfc59, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4] to archive 2024-11-24T02:43:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:36,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b4662286f08741d18748842f68c191b9 2024-11-24T02:43:36,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/bc12a2c4f77a40448e3d850023c4cfa2 2024-11-24T02:43:36,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1ee41f9c639345dcbb20e8c1f17cdfbf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1ee41f9c639345dcbb20e8c1f17cdfbf 2024-11-24T02:43:36,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f221e96a1f094b8aa783bfce4e0dab77 2024-11-24T02:43:36,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/e89cbd83545d42d293ad55d0f0535768 2024-11-24T02:43:36,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c349040bad954f909ba0988def9eec79 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c349040bad954f909ba0988def9eec79 2024-11-24T02:43:36,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/278c2339d9e944f0a72b7db1420e71bb 2024-11-24T02:43:36,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/c59458fef91849c4b8aaf7c2e84c16c9 2024-11-24T02:43:36,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/d9db48afc4a047fc92596b39400b4a1f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/d9db48afc4a047fc92596b39400b4a1f 2024-11-24T02:43:36,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/46ece3437f5748e2a667cc10d9494f27 2024-11-24T02:43:36,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/1a536f3f75c543b3bd33476841053817 2024-11-24T02:43:36,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/2ef1693e52bc44d0a7329bf0c0c44676 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/2ef1693e52bc44d0a7329bf0c0c44676 2024-11-24T02:43:36,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/8681b2c6110b403e8ef7911b63af1968 2024-11-24T02:43:36,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/757fe5eb041949719c7e22d30c7c9139 2024-11-24T02:43:36,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a0135945fb32493db96523ef85792b61 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a0135945fb32493db96523ef85792b61 2024-11-24T02:43:36,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/65e56fd0d9924b538ba06cbd2445e927 2024-11-24T02:43:36,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f2d61becc81143f28d539e4ae7da35e8 2024-11-24T02:43:36,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/3e848b82010c43dcbdb1b0b9b31d299d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/3e848b82010c43dcbdb1b0b9b31d299d 2024-11-24T02:43:36,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/90f819390d1d48eabf91b7b267c2b537 2024-11-24T02:43:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/39eb7afe6d444787bd27c833cc39d0e7 2024-11-24T02:43:36,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/b566da5cef17407785819ce43051e5b1 2024-11-24T02:43:36,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/102e5d241539471e88a25403a07cfc59 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/102e5d241539471e88a25403a07cfc59 2024-11-24T02:43:36,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/7a5b630e5b0c4687acd30d04ceaa8d6b 2024-11-24T02:43:36,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/43cb0f36f02046389a65ff07184a931f 2024-11-24T02:43:36,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/a131c1af37324fc6926443be34d497c4 2024-11-24T02:43:36,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/588ee4f39a094b6091ea535d2565f58d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/c14d017b52be438794227cdad0f65ab0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/414fbc1a18ae4cd1b6be31f02f8f3af0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/688b9a9fec0349779371f0218fcb4bdd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/058b329c79e340d1ae7bfc1d3967e014, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/872c114d53d9475b8d4426ee113f2346, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/85cc6596c0f342bb8e6b04907a979eac, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e] to archive 2024-11-24T02:43:36,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:43:36,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5fd2c0951475421182e2b88c0cc17b1f 2024-11-24T02:43:36,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/9ef5065d4c9140798446c6b66224b913 2024-11-24T02:43:36,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/588ee4f39a094b6091ea535d2565f58d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/588ee4f39a094b6091ea535d2565f58d 2024-11-24T02:43:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/5e9e5673ddad45ec93765f6a52a369fd 2024-11-24T02:43:36,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/a6c6bf39fc804cd8922d4f6cb2787bc7 2024-11-24T02:43:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/c14d017b52be438794227cdad0f65ab0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/c14d017b52be438794227cdad0f65ab0 2024-11-24T02:43:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3e235f0479bf4d09bf786d5c2b5149c8 2024-11-24T02:43:36,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/721357691930426cbd0603460ed5f032 2024-11-24T02:43:36,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/414fbc1a18ae4cd1b6be31f02f8f3af0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/414fbc1a18ae4cd1b6be31f02f8f3af0 2024-11-24T02:43:36,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8c6027ee9f644a2cbeb343a8f397037c 2024-11-24T02:43:36,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/3d4da6a481d346bbb31a6582004c37d5 2024-11-24T02:43:36,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/688b9a9fec0349779371f0218fcb4bdd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/688b9a9fec0349779371f0218fcb4bdd 2024-11-24T02:43:36,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/8e063a824d4b4d59b5225ca3b796240f 2024-11-24T02:43:36,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/d181f998b08d4270a11bdc291565db22 2024-11-24T02:43:36,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/058b329c79e340d1ae7bfc1d3967e014 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/058b329c79e340d1ae7bfc1d3967e014 2024-11-24T02:43:36,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/96d75c676e1c40e59ab7a4ec299bbb2d 2024-11-24T02:43:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/93cd71c79f2f4a51b45730fa6e42efa2 2024-11-24T02:43:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/872c114d53d9475b8d4426ee113f2346 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/872c114d53d9475b8d4426ee113f2346 2024-11-24T02:43:36,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/81394753c34148f990413bd429d457cc 2024-11-24T02:43:36,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/1eac0aadfc59415a98933f159ddc7c87 2024-11-24T02:43:36,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/14f9c360b088495594fa2334235d61b7 2024-11-24T02:43:36,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/85cc6596c0f342bb8e6b04907a979eac to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/85cc6596c0f342bb8e6b04907a979eac 2024-11-24T02:43:36,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/781bb64b8dda4ff0ab4ff66f37174bad 2024-11-24T02:43:36,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/21d1c09d75d848d19c7ce79f988a0bd4 2024-11-24T02:43:36,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/668f170a3d434e57b5894f426debfe2e 2024-11-24T02:43:36,878 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=1 2024-11-24T02:43:36,879 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. 2024-11-24T02:43:36,879 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for f40cba6b6b9bf91dffe1d7372b246258: Waiting for close lock at 1732416214435Running coprocessor pre-close hooks at 1732416214435Disabling compacts and flushes for region at 1732416214435Disabling writes for close at 1732416215514 (+1079 ms)Obtaining lock to block concurrent updates at 1732416215514Preparing flush snapshotting stores in f40cba6b6b9bf91dffe1d7372b246258 at 1732416215514Finished memstore snapshotting TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732416215515 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258. at 1732416215515Flushing f40cba6b6b9bf91dffe1d7372b246258/A: creating writer at 1732416215515Flushing f40cba6b6b9bf91dffe1d7372b246258/A: appending metadata at 1732416215517 (+2 ms)Flushing f40cba6b6b9bf91dffe1d7372b246258/A: closing flushed file at 1732416215517Flushing f40cba6b6b9bf91dffe1d7372b246258/B: creating writer at 1732416215927 (+410 ms)Flushing f40cba6b6b9bf91dffe1d7372b246258/B: appending metadata at 1732416215930 (+3 ms)Flushing f40cba6b6b9bf91dffe1d7372b246258/B: closing flushed file at 1732416215930Flushing f40cba6b6b9bf91dffe1d7372b246258/C: creating writer at 1732416216343 (+413 ms)Flushing f40cba6b6b9bf91dffe1d7372b246258/C: appending metadata at 1732416216347 (+4 ms)Flushing f40cba6b6b9bf91dffe1d7372b246258/C: closing flushed file at 1732416216347Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16ee91c5: reopening flushed file at 1732416216761 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39d80c29: reopening flushed file at 1732416216769 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59539cb7: reopening flushed file at 1732416216774 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f40cba6b6b9bf91dffe1d7372b246258 in 1265ms, sequenceid=417, compaction requested=true at 1732416216779 (+5 ms)Writing region close event to WAL at 1732416216875 (+96 ms)Running coprocessor post-close hooks at 1732416216879 (+4 ms)Closed at 1732416216879 2024-11-24T02:43:36,881 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:36,882 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=f40cba6b6b9bf91dffe1d7372b246258, regionState=CLOSED 2024-11-24T02:43:36,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:36,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T02:43:36,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure f40cba6b6b9bf91dffe1d7372b246258, server=7c69a60bd8f6,34251,1732416122955 in 2.6010 sec 2024-11-24T02:43:36,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-24T02:43:36,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f40cba6b6b9bf91dffe1d7372b246258, UNASSIGN in 2.6050 sec 2024-11-24T02:43:36,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T02:43:36,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6090 sec 2024-11-24T02:43:36,889 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416216889"}]},"ts":"1732416216889"} 2024-11-24T02:43:36,891 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:43:36,891 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:43:36,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6190 sec 2024-11-24T02:43:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:43:38,450 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:38,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:43:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,455 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:43:38,457 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:38,465 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/recovered.edits] 2024-11-24T02:43:38,471 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b345dbfa39594beaa320ac0cc884fed3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b345dbfa39594beaa320ac0cc884fed3 2024-11-24T02:43:38,473 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b389c941b05f4d7ba7020db36ef59379 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b389c941b05f4d7ba7020db36ef59379 2024-11-24T02:43:38,474 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b7c731db2ef34fa88c861a84e9a11a57 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/b7c731db2ef34fa88c861a84e9a11a57 2024-11-24T02:43:38,475 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/f51faf8f2c7d4123848fb661f9970f95 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/A/f51faf8f2c7d4123848fb661f9970f95 2024-11-24T02:43:38,477 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/49b698dfe65642179bee8a79167d41d6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/49b698dfe65642179bee8a79167d41d6 2024-11-24T02:43:38,479 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/6fd609989a164ccda04665102ad50bcd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/6fd609989a164ccda04665102ad50bcd 2024-11-24T02:43:38,480 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/ef3a6f38b3e547b8b9c30acf25f11fb0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/ef3a6f38b3e547b8b9c30acf25f11fb0 2024-11-24T02:43:38,481 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f0eda888748049998b99ab518f4775c9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/B/f0eda888748049998b99ab518f4775c9 2024-11-24T02:43:38,483 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/03828ce8a8c94021b9335b205a9d8030 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/03828ce8a8c94021b9335b205a9d8030 2024-11-24T02:43:38,484 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/51a5a45b17e44323a63cbbed9b2a155f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/51a5a45b17e44323a63cbbed9b2a155f 2024-11-24T02:43:38,486 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/7f15fa27adea43fe971c939a2345efcf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/7f15fa27adea43fe971c939a2345efcf 2024-11-24T02:43:38,488 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/daa7f79edb414b978b39e7e41be4d4de to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/C/daa7f79edb414b978b39e7e41be4d4de 2024-11-24T02:43:38,491 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/recovered.edits/420.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258/recovered.edits/420.seqid 2024-11-24T02:43:38,492 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/f40cba6b6b9bf91dffe1d7372b246258 2024-11-24T02:43:38,492 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:43:38,495 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,499 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:43:38,502 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:43:38,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:43:38,503 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416218503"}]},"ts":"9223372036854775807"} 2024-11-24T02:43:38,505 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:43:38,506 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f40cba6b6b9bf91dffe1d7372b246258, NAME => 'TestAcidGuarantees,,1732416191581.f40cba6b6b9bf91dffe1d7372b246258.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:43:38,506 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:43:38,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416218506"}]},"ts":"9223372036854775807"} 2024-11-24T02:43:38,508 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:43:38,509 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-24T02:43:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:43:38,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:43:38,559 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:38,567 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 248), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=10103 (was 10286) 2024-11-24T02:43:38,574 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=10103 2024-11-24T02:43:38,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:43:38,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:43:38,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:38,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:43:38,578 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:38,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-24T02:43:38,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:38,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:43:38,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742156_1332 (size=1061) 2024-11-24T02:43:38,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:38,987 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:43:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742157_1333 (size=53) 2024-11-24T02:43:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:39,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:43:39,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d4d30f9d55fca66379e8b4e679fdf056, disabling compactions & flushes 2024-11-24T02:43:39,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. after waiting 0 ms 2024-11-24T02:43:39,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,400 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d4d30f9d55fca66379e8b4e679fdf056: Waiting for close lock at 1732416219399Disabling compacts and flushes for region at 1732416219399Disabling writes for close at 1732416219399Writing region close event to WAL at 1732416219399Closed at 1732416219399 2024-11-24T02:43:39,402 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:43:39,403 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416219402"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416219402"}]},"ts":"1732416219402"} 2024-11-24T02:43:39,406 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:43:39,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:43:39,408 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416219408"}]},"ts":"1732416219408"} 2024-11-24T02:43:39,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:43:39,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, ASSIGN}] 2024-11-24T02:43:39,411 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, ASSIGN 2024-11-24T02:43:39,412 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:43:39,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:39,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, ASSIGN because future has completed 2024-11-24T02:43:39,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:39,731 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:43:39,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:43:39,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,733 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,734 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:39,735 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName A 2024-11-24T02:43:39,735 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:39,735 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:39,735 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,736 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:39,736 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName B 2024-11-24T02:43:39,736 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:39,737 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:39,737 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,738 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:39,738 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName C 2024-11-24T02:43:39,738 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:39,738 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:39,738 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,739 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,739 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,739 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,740 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,740 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,741 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:43:39,742 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,743 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:43:39,744 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened d4d30f9d55fca66379e8b4e679fdf056; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60143268, jitterRate=-0.10379546880722046}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:43:39,744 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:39,744 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for d4d30f9d55fca66379e8b4e679fdf056: Running coprocessor pre-open hook at 1732416219732Writing region info on filesystem at 1732416219732Initializing all the Stores at 1732416219733 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416219733Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416219733Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416219733Cleaning up temporary data from old regions at 1732416219740 (+7 ms)Running coprocessor post-open hooks at 1732416219744 (+4 ms)Region opened successfully at 1732416219744 2024-11-24T02:43:39,745 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., pid=87, masterSystemTime=1732416219723 2024-11-24T02:43:39,746 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,746 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:39,747 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:39,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:39,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-24T02:43:39,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 in 181 msec 2024-11-24T02:43:39,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-24T02:43:39,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, ASSIGN in 341 msec 2024-11-24T02:43:39,754 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:43:39,754 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416219754"}]},"ts":"1732416219754"} 2024-11-24T02:43:39,755 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:43:39,756 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:43:39,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-24T02:43:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-24T02:43:40,731 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:40,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29820261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:40,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:40,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:40,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1676f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:40,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36619e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:40,741 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:40,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:40,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:43:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:43:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@527392d8 2024-11-24T02:43:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:43:40,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:43:40,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:43:40,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:43:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:43:40,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742158_1334 (size=1097) 2024-11-24T02:43:41,213 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T02:43:41,213 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T02:43:41,218 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:43:41,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE}] 2024-11-24T02:43:41,222 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE 2024-11-24T02:43:41,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE because future has completed 2024-11-24T02:43:41,225 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:43:41,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:41,379 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,379 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:43:41,379 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing d4d30f9d55fca66379e8b4e679fdf056, disabling compactions & flushes 2024-11-24T02:43:41,379 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,380 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,380 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. after waiting 0 ms 2024-11-24T02:43:41,380 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,387 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:43:41,388 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,388 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for d4d30f9d55fca66379e8b4e679fdf056: Waiting for close lock at 1732416221379Running coprocessor pre-close hooks at 1732416221379Disabling compacts and flushes for region at 1732416221379Disabling writes for close at 1732416221380 (+1 ms)Writing region close event to WAL at 1732416221381 (+1 ms)Running coprocessor post-close hooks at 1732416221388 (+7 ms)Closed at 1732416221388 2024-11-24T02:43:41,388 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: d4d30f9d55fca66379e8b4e679fdf056 to self. 2024-11-24T02:43:41,390 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,391 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=CLOSED 2024-11-24T02:43:41,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:41,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T02:43:41,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 in 169 msec 2024-11-24T02:43:41,398 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=true 2024-11-24T02:43:41,549 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE because future has completed 2024-11-24T02:43:41,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:43:41,710 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,710 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:43:41,711 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,711 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:43:41,711 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,711 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,714 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,716 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:41,717 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName A 2024-11-24T02:43:41,719 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:41,720 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:41,721 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,722 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:41,722 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName B 2024-11-24T02:43:41,722 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:41,723 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:41,723 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,724 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:43:41,725 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4d30f9d55fca66379e8b4e679fdf056 columnFamilyName C 2024-11-24T02:43:41,725 DEBUG [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:41,725 INFO [StoreOpener-d4d30f9d55fca66379e8b4e679fdf056-1 {}] regionserver.HStore(327): Store=d4d30f9d55fca66379e8b4e679fdf056/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:43:41,725 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,725 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,726 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,727 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,728 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,728 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,728 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:43:41,729 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,730 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened d4d30f9d55fca66379e8b4e679fdf056; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65497665, jitterRate=-0.02400873601436615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:43:41,730 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,731 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for d4d30f9d55fca66379e8b4e679fdf056: Running coprocessor pre-open hook at 1732416221712Writing region info on filesystem at 1732416221712Initializing all the Stores at 1732416221713 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416221713Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416221714 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416221714Cleaning up temporary data from old regions at 1732416221728 (+14 ms)Running coprocessor post-open hooks at 1732416221730 (+2 ms)Region opened successfully at 1732416221731 (+1 ms) 2024-11-24T02:43:41,731 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., pid=92, masterSystemTime=1732416221705 2024-11-24T02:43:41,733 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,733 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,733 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:43:41,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-24T02:43:41,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 in 183 msec 2024-11-24T02:43:41,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T02:43:41,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, REOPEN/MOVE in 517 msec 2024-11-24T02:43:41,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T02:43:41,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-24T02:43:41,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 996 msec 2024-11-24T02:43:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T02:43:41,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:41,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c07883b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,748 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd3348d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,750 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17dd5182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d62a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,752 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8c30b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,754 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b8b984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6513d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1415f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f93014e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510f8e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765313d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,763 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616716ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,765 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e2d751f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164333e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@469d248f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,774 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1decd04a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b950b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e8c61ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,777 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db9485a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578ae440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@147a1007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1bf127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,781 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292b669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4d201d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,783 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca72fd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6a0796, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2b45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:43:41,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:43:41,786 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502e442e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:43:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:43:41,787 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:43:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a41d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:43:41,788 DEBUG [Thread-1525 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,788 DEBUG [Thread-1523 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,788 DEBUG [Thread-1538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,788 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,788 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,789 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,793 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,794 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,794 DEBUG [Thread-1534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,797 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:41,797 DEBUG [Thread-1534 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,798 DEBUG [Thread-1534 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-24T02:43:41,801 DEBUG [Thread-1536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:43:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:43:41,801 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:41,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:43:41,802 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:41,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:41,803 DEBUG [Thread-1536 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:43:41,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:43:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:43:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:41,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f35e5a1a4e9a4d4c9bf667cf96858744_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416221799/Put/seqid=0 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416221835, max timestamp=1732416221857Num uniques 859; 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416221835, max timestamp=1732416221857Num uniques 859; 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416221835, max timestamp=1732416221857Num uniques 859; 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:43:41,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:43:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742159_1335 (size=12154) 2024-11-24T02:43:41,866 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:41,870 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f35e5a1a4e9a4d4c9bf667cf96858744_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f35e5a1a4e9a4d4c9bf667cf96858744_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:41,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/99d790340b1e4b728a8fa7512a6d130a, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:41,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/99d790340b1e4b728a8fa7512a6d130a is 175, key is test_row_0/A:col10/1732416221799/Put/seqid=0 2024-11-24T02:43:41,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742160_1336 (size=30955) 2024-11-24T02:43:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416281891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416281895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416281895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416281909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:41,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416281911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:41,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:41,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:41,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416282019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416282019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416282019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416282019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416282040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416282239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416282239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416282239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416282240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416282259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/99d790340b1e4b728a8fa7512a6d130a 2024-11-24T02:43:42,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/a0521da824ac498ba1ca66d035b02d24 is 50, key is test_row_0/B:col10/1732416221799/Put/seqid=0 2024-11-24T02:43:42,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742161_1337 (size=12001) 2024-11-24T02:43:42,415 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:43:42,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416282548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416282559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416282559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416282559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416282569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:42,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:42,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/a0521da824ac498ba1ca66d035b02d24 2024-11-24T02:43:42,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/188dfb627656437cab37a6eeef953bf0 is 50, key is test_row_0/C:col10/1732416221799/Put/seqid=0 2024-11-24T02:43:42,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742162_1338 (size=12001) 2024-11-24T02:43:42,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:42,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:43,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:43,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:43,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416283059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416283079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416283079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:43,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416283079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416283088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:43,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/188dfb627656437cab37a6eeef953bf0 2024-11-24T02:43:43,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/99d790340b1e4b728a8fa7512a6d130a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a 2024-11-24T02:43:43,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a, entries=150, sequenceid=15, filesize=30.2 K 2024-11-24T02:43:43,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/a0521da824ac498ba1ca66d035b02d24 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24 2024-11-24T02:43:43,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:43:43,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/188dfb627656437cab37a6eeef953bf0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0 2024-11-24T02:43:43,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:43:43,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d4d30f9d55fca66379e8b4e679fdf056 in 1355ms, sequenceid=15, compaction requested=false 2024-11-24T02:43:43,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:43,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-24T02:43:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:43,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:43,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:43:43,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245bd1b8b993be4c2ead8445cc9cb30d3d_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416221909/Put/seqid=0 2024-11-24T02:43:43,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742163_1339 (size=12154) 2024-11-24T02:43:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:43,595 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245bd1b8b993be4c2ead8445cc9cb30d3d_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bd1b8b993be4c2ead8445cc9cb30d3d_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:43,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9d368ec8a55b40e5ae996dd6ac146ce5, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9d368ec8a55b40e5ae996dd6ac146ce5 is 175, key is test_row_0/A:col10/1732416221909/Put/seqid=0 2024-11-24T02:43:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742164_1340 (size=30955) 2024-11-24T02:43:43,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:44,000 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9d368ec8a55b40e5ae996dd6ac146ce5 2024-11-24T02:43:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/1884d7a25fbd450fab5504e42a7a05d4 is 50, key is test_row_0/B:col10/1732416221909/Put/seqid=0 2024-11-24T02:43:44,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742165_1341 (size=12001) 2024-11-24T02:43:44,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:44,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:44,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416284109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416284109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,117 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416284109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,117 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416284110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416284116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416284229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416284229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416284229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/1884d7a25fbd450fab5504e42a7a05d4 2024-11-24T02:43:44,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/f2385b50a4a240b086d19483f230fcd0 is 50, key is test_row_0/C:col10/1732416221909/Put/seqid=0 2024-11-24T02:43:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742166_1342 (size=12001) 2024-11-24T02:43:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416284439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416284439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416284439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416284749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416284749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416284749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:44,825 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/f2385b50a4a240b086d19483f230fcd0 2024-11-24T02:43:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9d368ec8a55b40e5ae996dd6ac146ce5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5 2024-11-24T02:43:44,832 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5, entries=150, sequenceid=49, filesize=30.2 K 2024-11-24T02:43:44,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/1884d7a25fbd450fab5504e42a7a05d4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4 2024-11-24T02:43:44,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T02:43:44,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/f2385b50a4a240b086d19483f230fcd0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0 2024-11-24T02:43:44,840 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T02:43:44,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for d4d30f9d55fca66379e8b4e679fdf056 in 1661ms, sequenceid=49, compaction requested=false 2024-11-24T02:43:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:44,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-24T02:43:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-24T02:43:44,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T02:43:44,843 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0400 sec 2024-11-24T02:43:44,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.0470 sec 2024-11-24T02:43:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:45,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:45,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247db3c3e5642a434d91071cdc932d35d5_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:45,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742167_1343 (size=14594) 2024-11-24T02:43:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416285320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416285326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416285328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416285439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416285439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416285439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416285648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416285649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416285649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:45,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247db3c3e5642a434d91071cdc932d35d5_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247db3c3e5642a434d91071cdc932d35d5_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:45,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1265c7e8b3974d899127b1b8c72869d0, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:45,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1265c7e8b3974d899127b1b8c72869d0 is 175, key is test_row_0/A:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:45,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742168_1344 (size=39549) 2024-11-24T02:43:45,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416285959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416285959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:45,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416285968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:45,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T02:43:45,979 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:45,979 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-24T02:43:45,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:45,981 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:45,981 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:45,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:46,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1265c7e8b3974d899127b1b8c72869d0 2024-11-24T02:43:46,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d6ab74cd60f54220a694a182be8e455e is 50, key is test_row_0/B:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:46,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742169_1345 (size=12001) 2024-11-24T02:43:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416286129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:46,132 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:46,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416286139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:46,146 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:46,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:46,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416286469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416286479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:46,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416286489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:46,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d6ab74cd60f54220a694a182be8e455e 2024-11-24T02:43:46,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/efea1927c98e40aeae71b917ec510d89 is 50, key is test_row_0/C:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:46,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742170_1346 (size=12001) 2024-11-24T02:43:46,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:46,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:46,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:46,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:46,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/efea1927c98e40aeae71b917ec510d89 2024-11-24T02:43:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1265c7e8b3974d899127b1b8c72869d0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0 2024-11-24T02:43:46,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0, entries=200, sequenceid=61, filesize=38.6 K 2024-11-24T02:43:46,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d6ab74cd60f54220a694a182be8e455e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e 2024-11-24T02:43:46,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T02:43:46,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/efea1927c98e40aeae71b917ec510d89 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89 2024-11-24T02:43:46,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T02:43:46,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d4d30f9d55fca66379e8b4e679fdf056 in 1661ms, sequenceid=61, compaction requested=true 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:46,934 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:46,934 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:46,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:46,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:46,935 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:46,935 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:43:46,935 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:43:46,936 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,936 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=99.1 K 2024-11-24T02:43:46,936 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,936 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0] 2024-11-24T02:43:46,936 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,936 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.2 K 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a0521da824ac498ba1ca66d035b02d24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416221797 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d790340b1e4b728a8fa7512a6d130a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416221797 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1884d7a25fbd450fab5504e42a7a05d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416221884 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d368ec8a55b40e5ae996dd6ac146ce5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416221884 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ab74cd60f54220a694a182be8e455e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:46,937 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1265c7e8b3974d899127b1b8c72869d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:46,943 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:46,944 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:46,945 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/20b19b2e745e4ed192a9a918d70efc79 is 50, key is test_row_0/B:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:46,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124cb1318cf71aa4418b80b60ea4df3fd25_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:46,948 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124cb1318cf71aa4418b80b60ea4df3fd25_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:46,948 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cb1318cf71aa4418b80b60ea4df3fd25_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:46,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742171_1347 (size=12104) 2024-11-24T02:43:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742172_1348 (size=4469) 2024-11-24T02:43:46,965 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#297 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:46,965 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/35bfc4d56dc14ed1abdfd0051c301ea3 is 175, key is test_row_0/A:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:46,967 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/20b19b2e745e4ed192a9a918d70efc79 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/20b19b2e745e4ed192a9a918d70efc79 2024-11-24T02:43:46,972 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into 20b19b2e745e4ed192a9a918d70efc79(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:46,972 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:46,972 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=13, startTime=1732416226934; duration=0sec 2024-11-24T02:43:46,973 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:46,973 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:43:46,973 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:46,979 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:46,979 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:43:46,979 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:46,979 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.2 K 2024-11-24T02:43:46,991 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 188dfb627656437cab37a6eeef953bf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416221797 2024-11-24T02:43:46,991 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f2385b50a4a240b086d19483f230fcd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732416221884 2024-11-24T02:43:46,991 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting efea1927c98e40aeae71b917ec510d89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:46,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742173_1349 (size=31058) 2024-11-24T02:43:47,006 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/35bfc4d56dc14ed1abdfd0051c301ea3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3 2024-11-24T02:43:47,017 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into 35bfc4d56dc14ed1abdfd0051c301ea3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:47,017 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:47,017 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=13, startTime=1732416226934; duration=0sec 2024-11-24T02:43:47,017 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:47,017 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:43:47,022 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:47,023 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/0ed215187a964a29b7dd9eed009ae3d2 is 50, key is test_row_0/C:col10/1732416224115/Put/seqid=0 2024-11-24T02:43:47,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-24T02:43:47,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:47,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:47,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742174_1350 (size=12104) 2024-11-24T02:43:47,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112480a0864ed23741db95e6d2037e617b8b_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416225314/Put/seqid=0 2024-11-24T02:43:47,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742175_1351 (size=12154) 2024-11-24T02:43:47,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:47,105 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112480a0864ed23741db95e6d2037e617b8b_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480a0864ed23741db95e6d2037e617b8b_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:47,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/b37ce573bd2e41a0b853a8f265b9078d, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:47,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/b37ce573bd2e41a0b853a8f265b9078d is 175, key is test_row_0/A:col10/1732416225314/Put/seqid=0 2024-11-24T02:43:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:47,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742176_1352 (size=30955) 2024-11-24T02:43:47,130 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/b37ce573bd2e41a0b853a8f265b9078d 2024-11-24T02:43:47,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/ae783136437943d380df296e0515f2a5 is 50, key is test_row_0/B:col10/1732416225314/Put/seqid=0 2024-11-24T02:43:47,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742177_1353 (size=12001) 2024-11-24T02:43:47,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/ae783136437943d380df296e0515f2a5 2024-11-24T02:43:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a1018f742cd04ba7a5cfda49e30a55fa is 50, key is test_row_0/C:col10/1732416225314/Put/seqid=0 2024-11-24T02:43:47,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742178_1354 (size=12001) 2024-11-24T02:43:47,474 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/0ed215187a964a29b7dd9eed009ae3d2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0ed215187a964a29b7dd9eed009ae3d2 2024-11-24T02:43:47,477 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into 0ed215187a964a29b7dd9eed009ae3d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:47,477 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:47,477 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=13, startTime=1732416226934; duration=0sec 2024-11-24T02:43:47,477 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:47,477 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:43:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:47,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416287504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416287505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416287509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,577 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a1018f742cd04ba7a5cfda49e30a55fa 2024-11-24T02:43:47,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/b37ce573bd2e41a0b853a8f265b9078d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d 2024-11-24T02:43:47,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d, entries=150, sequenceid=83, filesize=30.2 K 2024-11-24T02:43:47,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/ae783136437943d380df296e0515f2a5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5 2024-11-24T02:43:47,587 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T02:43:47,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a1018f742cd04ba7a5cfda49e30a55fa as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa 2024-11-24T02:43:47,592 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T02:43:47,593 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d4d30f9d55fca66379e8b4e679fdf056 in 543ms, sequenceid=83, compaction requested=false 2024-11-24T02:43:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:47,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-24T02:43:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-24T02:43:47,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-24T02:43:47,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-24T02:43:47,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6170 sec 2024-11-24T02:43:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:47,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:47,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243c3aa89b83a54b3292fb6b456a870628_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:47,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742179_1355 (size=14594) 2024-11-24T02:43:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416287668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416287676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416287678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416287789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416287789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416287789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416288009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416288009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416288009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:48,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243c3aa89b83a54b3292fb6b456a870628_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c3aa89b83a54b3292fb6b456a870628_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:48,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/efe14a76829742298577535e3440efb9, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:48,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/efe14a76829742298577535e3440efb9 is 175, key is test_row_0/A:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:48,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742180_1356 (size=39549) 2024-11-24T02:43:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-24T02:43:48,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:48,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-24T02:43:48,131 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:48,132 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:48,132 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:48,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:48,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:48,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416288319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416288319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416288319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:48,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:48,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:48,453 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/efe14a76829742298577535e3440efb9 2024-11-24T02:43:48,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7afb835bbb27451db98025588fd8cb55 is 50, key is test_row_0/B:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:48,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742181_1357 (size=12001) 2024-11-24T02:43:48,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:48,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:48,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:48,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416288829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416288829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416288838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:48,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:48,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7afb835bbb27451db98025588fd8cb55 2024-11-24T02:43:48,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/c0abf9c6ab604885b8c55a5755ee9357 is 50, key is test_row_0/C:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:48,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742182_1358 (size=12001) 2024-11-24T02:43:48,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:48,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:48,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:49,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:49,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/c0abf9c6ab604885b8c55a5755ee9357 2024-11-24T02:43:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/efe14a76829742298577535e3440efb9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9 2024-11-24T02:43:49,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9, entries=200, sequenceid=97, filesize=38.6 K 2024-11-24T02:43:49,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7afb835bbb27451db98025588fd8cb55 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55 2024-11-24T02:43:49,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T02:43:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/c0abf9c6ab604885b8c55a5755ee9357 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357 2024-11-24T02:43:49,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T02:43:49,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d4d30f9d55fca66379e8b4e679fdf056 in 1674ms, sequenceid=97, compaction requested=true 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:49,308 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:49,308 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:49,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:49,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:49,310 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:43:49,310 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:49,310 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,310 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:43:49,310 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,310 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/20b19b2e745e4ed192a9a918d70efc79, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.3 K 2024-11-24T02:43:49,310 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=99.2 K 2024-11-24T02:43:49,310 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,310 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9] 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b19b2e745e4ed192a9a918d70efc79, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35bfc4d56dc14ed1abdfd0051c301ea3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ae783136437943d380df296e0515f2a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732416225314 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b37ce573bd2e41a0b853a8f265b9078d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732416225314 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7afb835bbb27451db98025588fd8cb55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:49,311 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting efe14a76829742298577535e3440efb9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:49,316 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:49,317 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124127301765ec0408d9d5ec3ba6463279a_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:49,317 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:49,318 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/523f66c06e0242058068acb42a251114 is 50, key is test_row_0/B:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:49,319 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124127301765ec0408d9d5ec3ba6463279a_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:49,319 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124127301765ec0408d9d5ec3ba6463279a_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:49,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742183_1359 (size=12207) 2024-11-24T02:43:49,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742184_1360 (size=4469) 2024-11-24T02:43:49,335 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#307 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:49,335 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/0d2b4dc44d424112bfe70f61b6347de6 is 175, key is test_row_0/A:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:49,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742185_1361 (size=31161) 2024-11-24T02:43:49,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-24T02:43:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,353 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:43:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:49,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c35dc8762cbd470ba3df83f9cc3d9659_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:49,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742186_1362 (size=12154) 2024-11-24T02:43:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:49,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c35dc8762cbd470ba3df83f9cc3d9659_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c35dc8762cbd470ba3df83f9cc3d9659_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2930cffcfb5d4a99a0a9308558c7b829, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:49,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2930cffcfb5d4a99a0a9308558c7b829 is 175, key is test_row_0/A:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742187_1363 (size=30955) 2024-11-24T02:43:49,733 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/523f66c06e0242058068acb42a251114 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/523f66c06e0242058068acb42a251114 2024-11-24T02:43:49,736 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into 523f66c06e0242058068acb42a251114(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:49,736 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:49,736 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=13, startTime=1732416229308; duration=0sec 2024-11-24T02:43:49,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:49,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:43:49,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:49,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:49,737 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:43:49,737 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:49,737 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0ed215187a964a29b7dd9eed009ae3d2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.3 K 2024-11-24T02:43:49,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ed215187a964a29b7dd9eed009ae3d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732416224108 2024-11-24T02:43:49,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a1018f742cd04ba7a5cfda49e30a55fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732416225314 2024-11-24T02:43:49,738 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c0abf9c6ab604885b8c55a5755ee9357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:49,744 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:49,745 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/947c885b8fb94a7f9264c935024fc87a is 50, key is test_row_0/C:col10/1732416227633/Put/seqid=0 2024-11-24T02:43:49,746 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/0d2b4dc44d424112bfe70f61b6347de6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6 2024-11-24T02:43:49,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742188_1364 (size=12207) 2024-11-24T02:43:49,750 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into 0d2b4dc44d424112bfe70f61b6347de6(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:49,750 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:49,750 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=13, startTime=1732416229308; duration=0sec 2024-11-24T02:43:49,750 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:49,750 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:43:49,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2930cffcfb5d4a99a0a9308558c7b829 2024-11-24T02:43:49,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/2c906a174d684ecdbb7461de4d43084c is 50, key is test_row_0/B:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:49,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742189_1365 (size=12001) 2024-11-24T02:43:49,786 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/2c906a174d684ecdbb7461de4d43084c 2024-11-24T02:43:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ee6c63e00df94eb0861b01eb38c19f0a is 50, key is test_row_0/C:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742190_1366 (size=12001) 2024-11-24T02:43:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:49,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:49,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416289872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:49,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416289877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416289878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:49,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416289989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416289989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416289989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:49,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:49,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,153 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/947c885b8fb94a7f9264c935024fc87a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/947c885b8fb94a7f9264c935024fc87a 2024-11-24T02:43:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416290148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,156 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,160 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into 947c885b8fb94a7f9264c935024fc87a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:50,160 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:50,160 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=13, startTime=1732416229308; duration=0sec 2024-11-24T02:43:50,160 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:50,160 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:43:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416290169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,175 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:43:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ee6c63e00df94eb0861b01eb38c19f0a 2024-11-24T02:43:50,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2930cffcfb5d4a99a0a9308558c7b829 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829 2024-11-24T02:43:50,206 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829, entries=150, sequenceid=116, filesize=30.2 K 2024-11-24T02:43:50,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/2c906a174d684ecdbb7461de4d43084c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c 2024-11-24T02:43:50,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,210 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:43:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ee6c63e00df94eb0861b01eb38c19f0a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a 2024-11-24T02:43:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,214 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-24T02:43:50,215 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d4d30f9d55fca66379e8b4e679fdf056 in 862ms, sequenceid=116, compaction requested=false 2024-11-24T02:43:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-24T02:43:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-24T02:43:50,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:50,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:50,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T02:43:50,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0850 sec 2024-11-24T02:43:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.0910 sec 2024-11-24T02:43:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ccc3f9afd17145eaba5626ba4326889f_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_1/A:col10/1732416229876/Put/seqid=0 2024-11-24T02:43:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742191_1367 (size=12304) 2024-11-24T02:43:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,232 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ccc3f9afd17145eaba5626ba4326889f_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccc3f9afd17145eaba5626ba4326889f_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/751af5fe60c640879111342c39d4c135, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/751af5fe60c640879111342c39d4c135 is 175, key is test_row_1/A:col10/1732416229876/Put/seqid=0 2024-11-24T02:43:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742192_1368 (size=31101) 2024-11-24T02:43:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-24T02:43:50,279 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:50,280 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-24T02:43:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:50,281 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:50,282 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:50,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:50,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416290277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416290282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416290285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416290398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416290399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416290399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:50,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:50,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416290608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416290608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416290609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/751af5fe60c640879111342c39d4c135 2024-11-24T02:43:50,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/53fda9fff5604ad1bad03f088e0d412b is 50, key is test_row_1/B:col10/1732416229876/Put/seqid=0 2024-11-24T02:43:50,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742193_1369 (size=9757) 2024-11-24T02:43:50,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:50,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:50,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416290919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416290919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416290919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:51,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/53fda9fff5604ad1bad03f088e0d412b 2024-11-24T02:43:51,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/65168ca4a9ef41d587b1b5199dac84ca is 50, key is test_row_1/C:col10/1732416229876/Put/seqid=0 2024-11-24T02:43:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742194_1370 (size=9757) 2024-11-24T02:43:51,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416291429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416291429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:51,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416291439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:51,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:51,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/65168ca4a9ef41d587b1b5199dac84ca 2024-11-24T02:43:51,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/751af5fe60c640879111342c39d4c135 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135 2024-11-24T02:43:51,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135, entries=150, sequenceid=133, filesize=30.4 K 2024-11-24T02:43:51,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/53fda9fff5604ad1bad03f088e0d412b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b 2024-11-24T02:43:51,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b, entries=100, sequenceid=133, filesize=9.5 K 2024-11-24T02:43:51,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/65168ca4a9ef41d587b1b5199dac84ca as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca 2024-11-24T02:43:51,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca, entries=100, sequenceid=133, filesize=9.5 K 2024-11-24T02:43:51,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d4d30f9d55fca66379e8b4e679fdf056 in 1265ms, sequenceid=133, compaction requested=true 2024-11-24T02:43:51,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:51,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:51,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:43:51,482 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:43:51,482 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,482 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,482 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=91.0 K 2024-11-24T02:43:51,482 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/523f66c06e0242058068acb42a251114, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=33.2 K 2024-11-24T02:43:51,483 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135] 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 523f66c06e0242058068acb42a251114, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2b4dc44d424112bfe70f61b6347de6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c906a174d684ecdbb7461de4d43084c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416227665 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2930cffcfb5d4a99a0a9308558c7b829, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416227665 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 53fda9fff5604ad1bad03f088e0d412b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416229876 2024-11-24T02:43:51,483 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 751af5fe60c640879111342c39d4c135, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416229875 2024-11-24T02:43:51,488 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:51,490 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240dac25a5211e4cb3b38ddd05851f3ea5_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:51,491 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:51,492 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/cb57268d429e498fa40a02add3365165 is 50, key is test_row_0/B:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:51,492 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240dac25a5211e4cb3b38ddd05851f3ea5_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:51,492 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240dac25a5211e4cb3b38ddd05851f3ea5_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:51,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742195_1371 (size=12409) 2024-11-24T02:43:51,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,503 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:51,506 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/cb57268d429e498fa40a02add3365165 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/cb57268d429e498fa40a02add3365165 2024-11-24T02:43:51,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742196_1372 (size=4469) 2024-11-24T02:43:51,509 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#315 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:51,509 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2489b5254d0543de81e6c7be9feea4b4 is 175, key is test_row_0/A:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:51,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124de203f0b2fbd4321ac535437bd3cf112_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416230236/Put/seqid=0 2024-11-24T02:43:51,516 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into cb57268d429e498fa40a02add3365165(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:51,516 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:51,516 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=13, startTime=1732416231482; duration=0sec 2024-11-24T02:43:51,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:51,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:43:51,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:51,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742197_1373 (size=31470) 2024-11-24T02:43:51,518 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:51,519 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:43:51,519 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:51,519 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/947c885b8fb94a7f9264c935024fc87a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=33.2 K 2024-11-24T02:43:51,519 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 947c885b8fb94a7f9264c935024fc87a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732416227501 2024-11-24T02:43:51,520 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ee6c63e00df94eb0861b01eb38c19f0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732416227665 2024-11-24T02:43:51,520 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 65168ca4a9ef41d587b1b5199dac84ca, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416229876 2024-11-24T02:43:51,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742198_1374 (size=12304) 2024-11-24T02:43:51,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:51,527 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2489b5254d0543de81e6c7be9feea4b4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4 2024-11-24T02:43:51,528 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124de203f0b2fbd4321ac535437bd3cf112_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de203f0b2fbd4321ac535437bd3cf112_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:51,532 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:51,532 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/e91f2e8c84d84a13b9f97034ed4c408a is 50, key is test_row_0/C:col10/1732416227677/Put/seqid=0 2024-11-24T02:43:51,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/4fa1e55a19f9415290a1a002e0215d1e, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:51,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/4fa1e55a19f9415290a1a002e0215d1e is 175, key is test_row_0/A:col10/1732416230236/Put/seqid=0 2024-11-24T02:43:51,537 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into 2489b5254d0543de81e6c7be9feea4b4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:51,537 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:51,537 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=13, startTime=1732416231481; duration=0sec 2024-11-24T02:43:51,537 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:51,537 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:43:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742200_1376 (size=31105) 2024-11-24T02:43:51,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742199_1375 (size=12409) 2024-11-24T02:43:51,944 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/4fa1e55a19f9415290a1a002e0215d1e 2024-11-24T02:43:51,951 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/e91f2e8c84d84a13b9f97034ed4c408a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e91f2e8c84d84a13b9f97034ed4c408a 2024-11-24T02:43:51,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/59926db8dd6f4f6d81cc3bf4906e28a1 is 50, key is test_row_0/B:col10/1732416230236/Put/seqid=0 2024-11-24T02:43:51,960 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into e91f2e8c84d84a13b9f97034ed4c408a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:51,960 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:51,960 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=13, startTime=1732416231482; duration=0sec 2024-11-24T02:43:51,960 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:51,960 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:43:51,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742201_1377 (size=12151) 2024-11-24T02:43:52,366 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/59926db8dd6f4f6d81cc3bf4906e28a1 2024-11-24T02:43:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/bd85f839200d4b068fa3ac36e9a8bde6 is 50, key is test_row_0/C:col10/1732416230236/Put/seqid=0 2024-11-24T02:43:52,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742202_1378 (size=12151) 2024-11-24T02:43:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:52,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416292470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416292474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416292475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416292579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416292589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416292589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,775 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/bd85f839200d4b068fa3ac36e9a8bde6 2024-11-24T02:43:52,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/4fa1e55a19f9415290a1a002e0215d1e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e 2024-11-24T02:43:52,783 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e, entries=150, sequenceid=151, filesize=30.4 K 2024-11-24T02:43:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/59926db8dd6f4f6d81cc3bf4906e28a1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1 2024-11-24T02:43:52,786 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T02:43:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/bd85f839200d4b068fa3ac36e9a8bde6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6 2024-11-24T02:43:52,790 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6, entries=150, sequenceid=151, filesize=11.9 K 2024-11-24T02:43:52,792 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d4d30f9d55fca66379e8b4e679fdf056 in 1288ms, sequenceid=151, compaction requested=false 2024-11-24T02:43:52,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:52,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:52,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-24T02:43:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-24T02:43:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:52,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:52,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:52,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-24T02:43:52,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5110 sec 2024-11-24T02:43:52,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2cce12e0567475dab024c902a6b8285_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:52,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.5200 sec 2024-11-24T02:43:52,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742203_1379 (size=14794) 2024-11-24T02:43:52,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416292870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416292880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416292883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416292989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416292989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416292999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416293199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416293199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:53,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2cce12e0567475dab024c902a6b8285_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2cce12e0567475dab024c902a6b8285_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:53,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/655924dd16414702a59d316a8b64cf45, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:53,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/655924dd16414702a59d316a8b64cf45 is 175, key is test_row_0/A:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:53,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742204_1380 (size=39749) 2024-11-24T02:43:53,219 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/655924dd16414702a59d316a8b64cf45 2024-11-24T02:43:53,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/87b911fb099b4170abec2db46e294485 is 50, key is test_row_0/B:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:53,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416293219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742205_1381 (size=12151) 2024-11-24T02:43:53,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/87b911fb099b4170abec2db46e294485 2024-11-24T02:43:53,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 is 50, key is test_row_0/C:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:53,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742206_1382 (size=12151) 2024-11-24T02:43:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416293509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416293509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416293538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:53,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 2024-11-24T02:43:53,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/655924dd16414702a59d316a8b64cf45 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45 2024-11-24T02:43:53,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45, entries=200, sequenceid=169, filesize=38.8 K 2024-11-24T02:43:53,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/87b911fb099b4170abec2db46e294485 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485 2024-11-24T02:43:53,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T02:43:53,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 2024-11-24T02:43:53,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52, entries=150, sequenceid=169, filesize=11.9 K 2024-11-24T02:43:53,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d4d30f9d55fca66379e8b4e679fdf056 in 859ms, sequenceid=169, compaction requested=true 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:53,653 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:53,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:53,653 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102324 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:43:53,654 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:53,654 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:53,654 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/cb57268d429e498fa40a02add3365165, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.9 K 2024-11-24T02:43:53,654 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=99.9 K 2024-11-24T02:43:53,654 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45] 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting cb57268d429e498fa40a02add3365165, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416227677 2024-11-24T02:43:53,654 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2489b5254d0543de81e6c7be9feea4b4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416227677 2024-11-24T02:43:53,655 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 59926db8dd6f4f6d81cc3bf4906e28a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732416230236 2024-11-24T02:43:53,655 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa1e55a19f9415290a1a002e0215d1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732416230236 2024-11-24T02:43:53,655 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b911fb099b4170abec2db46e294485, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:43:53,655 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655924dd16414702a59d316a8b64cf45, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:43:53,660 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:53,661 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:53,661 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7efc7162e9e04b9dadd7b897dd7a477f is 50, key is test_row_0/B:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:53,662 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411247caa91f242014906950b875bc6788396_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:53,665 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411247caa91f242014906950b875bc6788396_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:53,665 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247caa91f242014906950b875bc6788396_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:53,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742207_1383 (size=12561) 2024-11-24T02:43:53,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742208_1384 (size=4469) 2024-11-24T02:43:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:54,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:43:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:54,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124601dc6959da14865a3b90b23e94ee148_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416234034/Put/seqid=0 2024-11-24T02:43:54,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742209_1385 (size=14794) 2024-11-24T02:43:54,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:54,051 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124601dc6959da14865a3b90b23e94ee148_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124601dc6959da14865a3b90b23e94ee148_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:54,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c35ce6f0444b4e3e98c3e0d3281a4c95, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:54,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c35ce6f0444b4e3e98c3e0d3281a4c95 is 175, key is test_row_0/A:col10/1732416234034/Put/seqid=0 2024-11-24T02:43:54,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742210_1386 (size=39749) 2024-11-24T02:43:54,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416294065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416294074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,079 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:54,080 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9db5fda9dd8f4064b4016efb718e14e4 is 175, key is test_row_0/A:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:54,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742211_1387 (size=31515) 2024-11-24T02:43:54,084 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7efc7162e9e04b9dadd7b897dd7a477f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7efc7162e9e04b9dadd7b897dd7a477f 2024-11-24T02:43:54,088 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/9db5fda9dd8f4064b4016efb718e14e4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4 2024-11-24T02:43:54,089 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into 7efc7162e9e04b9dadd7b897dd7a477f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:54,089 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:54,089 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=13, startTime=1732416233653; duration=0sec 2024-11-24T02:43:54,089 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:54,089 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:43:54,089 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:43:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416294076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,091 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:43:54,091 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:43:54,091 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,092 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e91f2e8c84d84a13b9f97034ed4c408a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=35.9 K 2024-11-24T02:43:54,092 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e91f2e8c84d84a13b9f97034ed4c408a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732416227677 2024-11-24T02:43:54,093 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting bd85f839200d4b068fa3ac36e9a8bde6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732416230236 2024-11-24T02:43:54,096 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a6682f1f1e4d4d8cbd66b4723c2d8c52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:43:54,096 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into 9db5fda9dd8f4064b4016efb718e14e4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:54,096 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:54,096 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=13, startTime=1732416233653; duration=0sec 2024-11-24T02:43:54,096 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:54,096 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:43:54,102 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:54,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/047f8f926a1d4c73963c9d0d52ce8031 is 50, key is test_row_0/C:col10/1732416232469/Put/seqid=0 2024-11-24T02:43:54,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742212_1388 (size=12561) 2024-11-24T02:43:54,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416294179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416294189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416294199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416294388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416294399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416294409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:43:54,449 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:54,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T02:43:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:54,450 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:54,451 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:54,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:54,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c35ce6f0444b4e3e98c3e0d3281a4c95 2024-11-24T02:43:54,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/f8e59a39a1ab4b9ca58b9048ee01ab0c is 50, key is test_row_0/B:col10/1732416234034/Put/seqid=0 2024-11-24T02:43:54,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742213_1389 (size=12151) 2024-11-24T02:43:54,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/f8e59a39a1ab4b9ca58b9048ee01ab0c 2024-11-24T02:43:54,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/6abb0e6cb8ab47f2a13b81c99b0ce046 is 50, key is test_row_0/C:col10/1732416234034/Put/seqid=0 2024-11-24T02:43:54,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742214_1390 (size=12151) 2024-11-24T02:43:54,513 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/047f8f926a1d4c73963c9d0d52ce8031 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/047f8f926a1d4c73963c9d0d52ce8031 2024-11-24T02:43:54,517 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into 047f8f926a1d4c73963c9d0d52ce8031(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:43:54,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:54,517 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=13, startTime=1732416233653; duration=0sec 2024-11-24T02:43:54,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:54,517 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:43:54,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:54,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T02:43:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416294698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416294708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416294729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:54,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:54,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T02:43:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:54,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/6abb0e6cb8ab47f2a13b81c99b0ce046 2024-11-24T02:43:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c35ce6f0444b4e3e98c3e0d3281a4c95 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95 2024-11-24T02:43:54,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95, entries=200, sequenceid=187, filesize=38.8 K 2024-11-24T02:43:54,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/f8e59a39a1ab4b9ca58b9048ee01ab0c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c 2024-11-24T02:43:54,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T02:43:54,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/6abb0e6cb8ab47f2a13b81c99b0ce046 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046 2024-11-24T02:43:54,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046, entries=150, sequenceid=187, filesize=11.9 K 2024-11-24T02:43:54,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d4d30f9d55fca66379e8b4e679fdf056 in 854ms, sequenceid=187, compaction requested=false 2024-11-24T02:43:54,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:54,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T02:43:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:54,909 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:54,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112440d04eb911b846188b37b23ebdcf3954_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416234073/Put/seqid=0 2024-11-24T02:43:54,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742215_1391 (size=12304) 2024-11-24T02:43:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:55,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416295274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416295284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416295285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:55,325 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112440d04eb911b846188b37b23ebdcf3954_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112440d04eb911b846188b37b23ebdcf3954_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:55,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1cccb70924044d789233a6f74ca8fd05, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1cccb70924044d789233a6f74ca8fd05 is 175, key is test_row_0/A:col10/1732416234073/Put/seqid=0 2024-11-24T02:43:55,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742216_1392 (size=31105) 2024-11-24T02:43:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416295389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416295399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416295399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:55,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416295609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416295619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416295619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,730 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1cccb70924044d789233a6f74ca8fd05 2024-11-24T02:43:55,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/3fada0afbd20479eb20ffb04144b9517 is 50, key is test_row_0/B:col10/1732416234073/Put/seqid=0 2024-11-24T02:43:55,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742217_1393 (size=12151) 2024-11-24T02:43:55,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416295929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416295939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416295939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:55,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:56,139 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/3fada0afbd20479eb20ffb04144b9517 2024-11-24T02:43:56,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/e028fe416ed9436b83118728d27cacfc is 50, key is test_row_0/C:col10/1732416234073/Put/seqid=0 2024-11-24T02:43:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742218_1394 (size=12151) 2024-11-24T02:43:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:56,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416296449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:56,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416296449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:56,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416296449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:56,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/e028fe416ed9436b83118728d27cacfc 2024-11-24T02:43:56,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/1cccb70924044d789233a6f74ca8fd05 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05 2024-11-24T02:43:56,557 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05, entries=150, sequenceid=204, filesize=30.4 K 2024-11-24T02:43:56,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/3fada0afbd20479eb20ffb04144b9517 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517 2024-11-24T02:43:56,561 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T02:43:56,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/e028fe416ed9436b83118728d27cacfc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc 2024-11-24T02:43:56,565 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T02:43:56,566 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d4d30f9d55fca66379e8b4e679fdf056 in 1657ms, sequenceid=204, compaction requested=true 2024-11-24T02:43:56,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:56,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:56,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T02:43:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T02:43:56,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T02:43:56,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1170 sec 2024-11-24T02:43:56,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1210 sec 2024-11-24T02:43:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:43:56,599 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:56,599 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T02:43:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:56,601 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:56,601 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:56,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:56,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:43:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:56,753 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:56,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112431914b78495344858d155f6fb8a47589_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416235282/Put/seqid=0 2024-11-24T02:43:56,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742219_1395 (size=12304) 2024-11-24T02:43:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:57,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:57,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112431914b78495344858d155f6fb8a47589_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431914b78495344858d155f6fb8a47589_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:57,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ce6ba8ea255846dbb5753d47b24873f8, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:57,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ce6ba8ea255846dbb5753d47b24873f8 is 175, key is test_row_0/A:col10/1732416235282/Put/seqid=0 2024-11-24T02:43:57,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742220_1396 (size=31105) 2024-11-24T02:43:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:57,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416297493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416297494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416297495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,571 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ce6ba8ea255846dbb5753d47b24873f8 2024-11-24T02:43:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/8ccbcea2096a4d3aa9e7201d066b8f70 is 50, key is test_row_0/B:col10/1732416235282/Put/seqid=0 2024-11-24T02:43:57,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742221_1397 (size=12151) 2024-11-24T02:43:57,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416297609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416297609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416297609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:57,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416297829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416297829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416297829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:57,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/8ccbcea2096a4d3aa9e7201d066b8f70 2024-11-24T02:43:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/0a64ee927681468c8118690f2aa37ac0 is 50, key is test_row_0/C:col10/1732416235282/Put/seqid=0 2024-11-24T02:43:57,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742222_1398 (size=12151) 2024-11-24T02:43:57,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/0a64ee927681468c8118690f2aa37ac0 2024-11-24T02:43:57,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ce6ba8ea255846dbb5753d47b24873f8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8 2024-11-24T02:43:57,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8, entries=150, sequenceid=222, filesize=30.4 K 2024-11-24T02:43:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/8ccbcea2096a4d3aa9e7201d066b8f70 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70 2024-11-24T02:43:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70, entries=150, sequenceid=222, filesize=11.9 K 2024-11-24T02:43:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/0a64ee927681468c8118690f2aa37ac0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0 2024-11-24T02:43:58,007 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0, entries=150, sequenceid=222, filesize=11.9 K 2024-11-24T02:43:58,008 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d4d30f9d55fca66379e8b4e679fdf056 in 1255ms, sequenceid=222, compaction requested=true 2024-11-24T02:43:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:58,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T02:43:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T02:43:58,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T02:43:58,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-24T02:43:58,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4120 sec 2024-11-24T02:43:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:58,146 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:43:58,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:58,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:58,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e1305cf7fe704c20a2f21814246b9ad1_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416238145/Put/seqid=0 2024-11-24T02:43:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742223_1399 (size=17284) 2024-11-24T02:43:58,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416298176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416298183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416298186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416298299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416298299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416298299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416298509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416298509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416298519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:43:58,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e1305cf7fe704c20a2f21814246b9ad1_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1305cf7fe704c20a2f21814246b9ad1_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:43:58,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/65ad5b22ccf7454e9cade763b75c3247, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:58,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/65ad5b22ccf7454e9cade763b75c3247 is 175, key is test_row_0/A:col10/1732416238145/Put/seqid=0 2024-11-24T02:43:58,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742224_1400 (size=48389) 2024-11-24T02:43:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:43:58,749 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:43:58,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:43:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T02:43:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:43:58,751 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:43:58,752 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:43:58,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:43:58,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416298829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416298829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416298839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:43:58,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:58,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:58,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:58,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/65ad5b22ccf7454e9cade763b75c3247 2024-11-24T02:43:58,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/615ae972b6d4433fa3dc9b0cda81ee28 is 50, key is test_row_0/B:col10/1732416238145/Put/seqid=0 2024-11-24T02:43:58,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742225_1401 (size=12151) 2024-11-24T02:43:59,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:43:59,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416299349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416299349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:43:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416299359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:43:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:43:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:43:59,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/615ae972b6d4433fa3dc9b0cda81ee28 2024-11-24T02:43:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:43:59,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/fb6898934f304b238acc7276bf2dd838 is 50, key is test_row_0/C:col10/1732416238145/Put/seqid=0 2024-11-24T02:43:59,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742226_1402 (size=12151) 2024-11-24T02:43:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:43:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:43:59,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/fb6898934f304b238acc7276bf2dd838 2024-11-24T02:43:59,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/65ad5b22ccf7454e9cade763b75c3247 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247 2024-11-24T02:43:59,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247, entries=250, sequenceid=238, filesize=47.3 K 2024-11-24T02:43:59,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/615ae972b6d4433fa3dc9b0cda81ee28 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28 2024-11-24T02:43:59,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28, entries=150, sequenceid=238, filesize=11.9 K 2024-11-24T02:43:59,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/fb6898934f304b238acc7276bf2dd838 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838 2024-11-24T02:43:59,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838, entries=150, sequenceid=238, filesize=11.9 K 2024-11-24T02:43:59,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d4d30f9d55fca66379e8b4e679fdf056 in 1651ms, sequenceid=238, compaction requested=true 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:59,797 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:43:59,797 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:43:59,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:43:59,798 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:43:59,798 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:43:59,798 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,799 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7efc7162e9e04b9dadd7b897dd7a477f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=59.7 K 2024-11-24T02:43:59,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 181863 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:43:59,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:43:59,799 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,799 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=177.6 K 2024-11-24T02:43:59,799 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247] 2024-11-24T02:43:59,799 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7efc7162e9e04b9dadd7b897dd7a477f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:43:59,800 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e59a39a1ab4b9ca58b9048ee01ab0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416232867 2024-11-24T02:43:59,800 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db5fda9dd8f4064b4016efb718e14e4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:43:59,802 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fada0afbd20479eb20ffb04144b9517, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416234063 2024-11-24T02:43:59,802 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c35ce6f0444b4e3e98c3e0d3281a4c95, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416232867 2024-11-24T02:43:59,803 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccbcea2096a4d3aa9e7201d066b8f70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732416235272 2024-11-24T02:43:59,803 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cccb70924044d789233a6f74ca8fd05, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416234063 2024-11-24T02:43:59,803 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 615ae972b6d4433fa3dc9b0cda81ee28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:43:59,803 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce6ba8ea255846dbb5753d47b24873f8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732416235272 2024-11-24T02:43:59,803 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ad5b22ccf7454e9cade763b75c3247, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:43:59,809 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:59,810 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:43:59,810 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/fddd07cb1c33483c8f4b597a848df525 is 50, key is test_row_0/B:col10/1732416238145/Put/seqid=0 2024-11-24T02:43:59,816 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124bbaae79744484d0885f05b5b5afd4fe8_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:43:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:43:59,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T02:43:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:43:59,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:43:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:43:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:43:59,820 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124bbaae79744484d0885f05b5b5afd4fe8_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:59,820 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bbaae79744484d0885f05b5b5afd4fe8_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:43:59,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742227_1403 (size=12731) 2024-11-24T02:43:59,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dabd43694c77474f9f3987242551f5cb_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416238173/Put/seqid=0 2024-11-24T02:43:59,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742228_1404 (size=4469) 2024-11-24T02:43:59,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742229_1405 (size=12304) 2024-11-24T02:43:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:44:00,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. as already flushing 2024-11-24T02:44:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:00,236 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/fddd07cb1c33483c8f4b597a848df525 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/fddd07cb1c33483c8f4b597a848df525 2024-11-24T02:44:00,239 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into fddd07cb1c33483c8f4b597a848df525(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:00,240 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:00,240 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=11, startTime=1732416239797; duration=0sec 2024-11-24T02:44:00,240 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:00,240 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:44:00,240 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:44:00,240 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:00,241 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2c17a3045795481f9495e066d552403f is 175, key is test_row_0/A:col10/1732416238145/Put/seqid=0 2024-11-24T02:44:00,241 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61165 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:44:00,241 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:44:00,241 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:00,241 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/047f8f926a1d4c73963c9d0d52ce8031, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=59.7 K 2024-11-24T02:44:00,242 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 047f8f926a1d4c73963c9d0d52ce8031, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416232469 2024-11-24T02:44:00,242 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6abb0e6cb8ab47f2a13b81c99b0ce046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732416232867 2024-11-24T02:44:00,242 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e028fe416ed9436b83118728d27cacfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416234063 2024-11-24T02:44:00,242 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a64ee927681468c8118690f2aa37ac0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1732416235272 2024-11-24T02:44:00,243 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6898934f304b238acc7276bf2dd838, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:44:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742230_1406 (size=31685) 2024-11-24T02:44:00,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:00,250 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:00,250 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/8874d926ff4b45b1b772580dccc99561 is 50, key is test_row_0/C:col10/1732416238145/Put/seqid=0 2024-11-24T02:44:00,252 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dabd43694c77474f9f3987242551f5cb_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dabd43694c77474f9f3987242551f5cb_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:00,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/3848e9f1c151460fb50901aa7f1cc710, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:00,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742231_1407 (size=12731) 2024-11-24T02:44:00,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/3848e9f1c151460fb50901aa7f1cc710 is 175, key is test_row_0/A:col10/1732416238173/Put/seqid=0 2024-11-24T02:44:00,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742232_1408 (size=31105) 2024-11-24T02:44:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416300304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,315 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416300310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416300359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416300359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,370 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,370 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416300379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,387 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416300419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416300429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416300628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416300639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,648 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/2c17a3045795481f9495e066d552403f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f 2024-11-24T02:44:00,651 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into 2c17a3045795481f9495e066d552403f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:00,651 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:00,651 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=11, startTime=1732416239797; duration=0sec 2024-11-24T02:44:00,651 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:00,651 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:44:00,657 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/8874d926ff4b45b1b772580dccc99561 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/8874d926ff4b45b1b772580dccc99561 2024-11-24T02:44:00,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/3848e9f1c151460fb50901aa7f1cc710 2024-11-24T02:44:00,661 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into 8874d926ff4b45b1b772580dccc99561(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:00,661 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:00,661 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=11, startTime=1732416239797; duration=0sec 2024-11-24T02:44:00,661 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:00,661 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:44:00,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d94617fb7cbe44f2ae60abc180ed8689 is 50, key is test_row_0/B:col10/1732416238173/Put/seqid=0 2024-11-24T02:44:00,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742233_1409 (size=12151) 2024-11-24T02:44:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:44:00,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416300939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:00,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416300949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,067 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d94617fb7cbe44f2ae60abc180ed8689 2024-11-24T02:44:01,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/782318b53d2543fc88cf8c594f12b70d is 50, key is test_row_0/C:col10/1732416238173/Put/seqid=0 2024-11-24T02:44:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742234_1410 (size=12151) 2024-11-24T02:44:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/782318b53d2543fc88cf8c594f12b70d 2024-11-24T02:44:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/3848e9f1c151460fb50901aa7f1cc710 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710 2024-11-24T02:44:01,132 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710, entries=150, sequenceid=254, filesize=30.4 K 2024-11-24T02:44:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/d94617fb7cbe44f2ae60abc180ed8689 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689 2024-11-24T02:44:01,136 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T02:44:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/782318b53d2543fc88cf8c594f12b70d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d 2024-11-24T02:44:01,140 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T02:44:01,141 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d4d30f9d55fca66379e8b4e679fdf056 in 1321ms, sequenceid=254, compaction requested=false 2024-11-24T02:44:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:01,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T02:44:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T02:44:01,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T02:44:01,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3900 sec 2024-11-24T02:44:01,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3930 sec 2024-11-24T02:44:01,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:44:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:01,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:44:01,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:01,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112469980737d3424010b398e8cf408e024f_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742235_1411 (size=14994) 2024-11-24T02:44:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416301493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416301500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416301608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416301609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,801 INFO [Thread-1536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1536. 2024-11-24T02:44:01,801 DEBUG [Thread-1536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:01,801 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,801 DEBUG [Thread-1536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:01,803 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-24T02:44:01,803 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:01,803 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,803 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:01,803 INFO [Thread-1534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1534. 2024-11-24T02:44:01,803 DEBUG [Thread-1534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:01,803 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,803 DEBUG [Thread-1534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:01,811 INFO [Thread-1538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1538. 2024-11-24T02:44:01,811 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-24T02:44:01,811 DEBUG [Thread-1538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:01,811 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:01,811 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,811 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,811 DEBUG [Thread-1538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,811 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:01,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:01,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416301819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416301829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:01,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:01,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:01,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112469980737d3424010b398e8cf408e024f_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469980737d3424010b398e8cf408e024f_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:01,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ff1c5cb137d444f49df88cec180c2b5d, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:01,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ff1c5cb137d444f49df88cec180c2b5d is 175, key is test_row_0/A:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:01,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742236_1412 (size=39949) 2024-11-24T02:44:02,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416302129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416302140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ff1c5cb137d444f49df88cec180c2b5d 2024-11-24T02:44:02,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7769cc85528b4d22aeb7f17dd9d9a22e is 50, key is test_row_0/B:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:02,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742237_1413 (size=12301) 2024-11-24T02:44:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46994 deadline: 1732416302379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,381 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47016 deadline: 1732416302390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,391 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47014 deadline: 1732416302399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,401 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in d4d30f9d55fca66379e8b4e679fdf056 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47008 deadline: 1732416302640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47002 deadline: 1732416302649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:02,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7769cc85528b4d22aeb7f17dd9d9a22e 2024-11-24T02:44:02,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ea55470a446b4a32a699b01aa33a114c is 50, key is test_row_0/C:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:02,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742238_1414 (size=12301) 2024-11-24T02:44:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:44:02,910 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:03,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ea55470a446b4a32a699b01aa33a114c 2024-11-24T02:44:03,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ff1c5cb137d444f49df88cec180c2b5d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d 2024-11-24T02:44:03,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d, entries=200, sequenceid=274, filesize=39.0 K 2024-11-24T02:44:03,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/7769cc85528b4d22aeb7f17dd9d9a22e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e 2024-11-24T02:44:03,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:44:03,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/ea55470a446b4a32a699b01aa33a114c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c 2024-11-24T02:44:03,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:44:03,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d4d30f9d55fca66379e8b4e679fdf056 in 1688ms, sequenceid=274, compaction requested=true 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:03,140 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d4d30f9d55fca66379e8b4e679fdf056:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:03,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:03,140 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:03,141 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:03,141 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:03,141 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/A is initiating minor compaction (all files) 2024-11-24T02:44:03,141 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/B is initiating minor compaction (all files) 2024-11-24T02:44:03,141 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/A in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:03,141 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/B in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:03,141 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=100.3 K 2024-11-24T02:44:03,141 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/fddd07cb1c33483c8f4b597a848df525, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=36.3 K 2024-11-24T02:44:03,141 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:03,141 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d] 2024-11-24T02:44:03,142 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fddd07cb1c33483c8f4b597a848df525, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:44:03,142 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c17a3045795481f9495e066d552403f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:44:03,142 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d94617fb7cbe44f2ae60abc180ed8689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416238173 2024-11-24T02:44:03,142 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3848e9f1c151460fb50901aa7f1cc710, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416238173 2024-11-24T02:44:03,143 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 7769cc85528b4d22aeb7f17dd9d9a22e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416240301 2024-11-24T02:44:03,143 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1c5cb137d444f49df88cec180c2b5d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416240301 2024-11-24T02:44:03,149 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:03,150 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:03,150 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f0b629ac95ca4c689bbf76e67a3ea4f8_d4d30f9d55fca66379e8b4e679fdf056 store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:03,150 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/e17802c72b29411086e1e09f8694b483 is 50, key is test_row_0/B:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:03,154 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f0b629ac95ca4c689bbf76e67a3ea4f8_d4d30f9d55fca66379e8b4e679fdf056, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:03,154 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f0b629ac95ca4c689bbf76e67a3ea4f8_d4d30f9d55fca66379e8b4e679fdf056 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742239_1415 (size=12983) 2024-11-24T02:44:03,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742240_1416 (size=4469) 2024-11-24T02:44:03,559 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#A#compaction#349 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:03,560 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c45e1452a5084fa6b26d8d94e458c91a is 175, key is test_row_0/A:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:03,565 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/e17802c72b29411086e1e09f8694b483 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/e17802c72b29411086e1e09f8694b483 2024-11-24T02:44:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742241_1417 (size=31937) 2024-11-24T02:44:03,570 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/B of d4d30f9d55fca66379e8b4e679fdf056 into e17802c72b29411086e1e09f8694b483(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:03,570 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:03,570 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/B, priority=13, startTime=1732416243140; duration=0sec 2024-11-24T02:44:03,570 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:03,570 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:B 2024-11-24T02:44:03,570 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:03,571 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:03,571 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): d4d30f9d55fca66379e8b4e679fdf056/C is initiating minor compaction (all files) 2024-11-24T02:44:03,571 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d4d30f9d55fca66379e8b4e679fdf056/C in TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:03,571 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/8874d926ff4b45b1b772580dccc99561, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp, totalSize=36.3 K 2024-11-24T02:44:03,572 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8874d926ff4b45b1b772580dccc99561, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732416237491 2024-11-24T02:44:03,572 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 782318b53d2543fc88cf8c594f12b70d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416238173 2024-11-24T02:44:03,572 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ea55470a446b4a32a699b01aa33a114c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416240301 2024-11-24T02:44:03,579 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d4d30f9d55fca66379e8b4e679fdf056#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:03,579 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/10ce90c22dbe462d956f989a9c8b7340 is 50, key is test_row_0/C:col10/1732416241450/Put/seqid=0 2024-11-24T02:44:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742242_1418 (size=12983) 2024-11-24T02:44:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:03,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T02:44:03,651 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-11-24T02:44:03,652 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:44:03,652 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:44:03,652 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:03,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:44:03,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:03,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b74b9cecf2b9410eb499de9cd190f16c_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416241498/Put/seqid=0 2024-11-24T02:44:03,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742243_1419 (size=12454) 2024-11-24T02:44:03,670 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-11-24T02:44:03,670 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:03,670 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:03,670 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:03,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:03,977 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/c45e1452a5084fa6b26d8d94e458c91a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c45e1452a5084fa6b26d8d94e458c91a 2024-11-24T02:44:03,985 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/A of d4d30f9d55fca66379e8b4e679fdf056 into c45e1452a5084fa6b26d8d94e458c91a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:03,985 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:03,985 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/A, priority=13, startTime=1732416243140; duration=0sec 2024-11-24T02:44:03,985 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:03,985 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:A 2024-11-24T02:44:03,988 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/10ce90c22dbe462d956f989a9c8b7340 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/10ce90c22dbe462d956f989a9c8b7340 2024-11-24T02:44:03,991 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d4d30f9d55fca66379e8b4e679fdf056/C of d4d30f9d55fca66379e8b4e679fdf056 into 10ce90c22dbe462d956f989a9c8b7340(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:03,992 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:03,992 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., storeName=d4d30f9d55fca66379e8b4e679fdf056/C, priority=13, startTime=1732416243140; duration=0sec 2024-11-24T02:44:03,992 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:03,992 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d4d30f9d55fca66379e8b4e679fdf056:C 2024-11-24T02:44:04,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:04,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b74b9cecf2b9410eb499de9cd190f16c_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b74b9cecf2b9410eb499de9cd190f16c_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:04,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ad5e45044fc546819bbaafd8b113aefd, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:04,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ad5e45044fc546819bbaafd8b113aefd is 175, key is test_row_0/A:col10/1732416241498/Put/seqid=0 2024-11-24T02:44:04,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742244_1420 (size=31255) 2024-11-24T02:44:04,484 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ad5e45044fc546819bbaafd8b113aefd 2024-11-24T02:44:04,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/0204a5806f9e4e87af59dc6d13aff033 is 50, key is test_row_0/B:col10/1732416241498/Put/seqid=0 2024-11-24T02:44:04,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742245_1421 (size=12301) 2024-11-24T02:44:04,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/0204a5806f9e4e87af59dc6d13aff033 2024-11-24T02:44:04,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/973d6429df9849aea812429b70f88cc2 is 50, key is test_row_0/C:col10/1732416241498/Put/seqid=0 2024-11-24T02:44:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742246_1422 (size=12301) 2024-11-24T02:44:05,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/973d6429df9849aea812429b70f88cc2 2024-11-24T02:44:05,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/ad5e45044fc546819bbaafd8b113aefd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ad5e45044fc546819bbaafd8b113aefd 2024-11-24T02:44:05,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ad5e45044fc546819bbaafd8b113aefd, entries=150, sequenceid=291, filesize=30.5 K 2024-11-24T02:44:05,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/0204a5806f9e4e87af59dc6d13aff033 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/0204a5806f9e4e87af59dc6d13aff033 2024-11-24T02:44:05,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/0204a5806f9e4e87af59dc6d13aff033, entries=150, sequenceid=291, filesize=12.0 K 2024-11-24T02:44:05,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/973d6429df9849aea812429b70f88cc2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/973d6429df9849aea812429b70f88cc2 2024-11-24T02:44:05,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/973d6429df9849aea812429b70f88cc2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-24T02:44:05,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for d4d30f9d55fca66379e8b4e679fdf056 in 1686ms, sequenceid=291, compaction requested=false 2024-11-24T02:44:05,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d4d30f9d55fca66379e8b4e679fdf056: 2024-11-24T02:44:05,935 WARN [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-24T02:44:05,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d] to archive 2024-11-24T02:44:05,941 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:05,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/99d790340b1e4b728a8fa7512a6d130a 2024-11-24T02:44:05,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9d368ec8a55b40e5ae996dd6ac146ce5 2024-11-24T02:44:05,944 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1265c7e8b3974d899127b1b8c72869d0 2024-11-24T02:44:05,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/35bfc4d56dc14ed1abdfd0051c301ea3 2024-11-24T02:44:05,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/b37ce573bd2e41a0b853a8f265b9078d 2024-11-24T02:44:05,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/efe14a76829742298577535e3440efb9 2024-11-24T02:44:05,948 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/0d2b4dc44d424112bfe70f61b6347de6 2024-11-24T02:44:05,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2930cffcfb5d4a99a0a9308558c7b829 2024-11-24T02:44:05,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2489b5254d0543de81e6c7be9feea4b4 2024-11-24T02:44:05,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/751af5fe60c640879111342c39d4c135 2024-11-24T02:44:05,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/4fa1e55a19f9415290a1a002e0215d1e 2024-11-24T02:44:05,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/655924dd16414702a59d316a8b64cf45 2024-11-24T02:44:05,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/9db5fda9dd8f4064b4016efb718e14e4 2024-11-24T02:44:05,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c35ce6f0444b4e3e98c3e0d3281a4c95 2024-11-24T02:44:05,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/1cccb70924044d789233a6f74ca8fd05 2024-11-24T02:44:05,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ce6ba8ea255846dbb5753d47b24873f8 2024-11-24T02:44:05,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/65ad5b22ccf7454e9cade763b75c3247 2024-11-24T02:44:05,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/2c17a3045795481f9495e066d552403f 2024-11-24T02:44:05,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/3848e9f1c151460fb50901aa7f1cc710 2024-11-24T02:44:05,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ff1c5cb137d444f49df88cec180c2b5d 2024-11-24T02:44:05,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/20b19b2e745e4ed192a9a918d70efc79, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/523f66c06e0242058068acb42a251114, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/cb57268d429e498fa40a02add3365165, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7efc7162e9e04b9dadd7b897dd7a477f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/fddd07cb1c33483c8f4b597a848df525, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e] to archive 2024-11-24T02:44:05,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:05,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/a0521da824ac498ba1ca66d035b02d24 2024-11-24T02:44:05,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/1884d7a25fbd450fab5504e42a7a05d4 2024-11-24T02:44:05,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/20b19b2e745e4ed192a9a918d70efc79 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/20b19b2e745e4ed192a9a918d70efc79 2024-11-24T02:44:05,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d6ab74cd60f54220a694a182be8e455e 2024-11-24T02:44:05,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/ae783136437943d380df296e0515f2a5 2024-11-24T02:44:05,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/523f66c06e0242058068acb42a251114 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/523f66c06e0242058068acb42a251114 2024-11-24T02:44:05,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7afb835bbb27451db98025588fd8cb55 2024-11-24T02:44:05,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/2c906a174d684ecdbb7461de4d43084c 2024-11-24T02:44:05,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/cb57268d429e498fa40a02add3365165 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/cb57268d429e498fa40a02add3365165 2024-11-24T02:44:05,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/53fda9fff5604ad1bad03f088e0d412b 2024-11-24T02:44:05,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/59926db8dd6f4f6d81cc3bf4906e28a1 2024-11-24T02:44:05,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7efc7162e9e04b9dadd7b897dd7a477f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7efc7162e9e04b9dadd7b897dd7a477f 2024-11-24T02:44:05,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/87b911fb099b4170abec2db46e294485 2024-11-24T02:44:05,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/f8e59a39a1ab4b9ca58b9048ee01ab0c 2024-11-24T02:44:05,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/3fada0afbd20479eb20ffb04144b9517 2024-11-24T02:44:05,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/8ccbcea2096a4d3aa9e7201d066b8f70 2024-11-24T02:44:05,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/fddd07cb1c33483c8f4b597a848df525 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/fddd07cb1c33483c8f4b597a848df525 2024-11-24T02:44:05,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/615ae972b6d4433fa3dc9b0cda81ee28 2024-11-24T02:44:05,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/d94617fb7cbe44f2ae60abc180ed8689 2024-11-24T02:44:05,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/7769cc85528b4d22aeb7f17dd9d9a22e 2024-11-24T02:44:05,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0ed215187a964a29b7dd9eed009ae3d2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/947c885b8fb94a7f9264c935024fc87a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e91f2e8c84d84a13b9f97034ed4c408a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/047f8f926a1d4c73963c9d0d52ce8031, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/8874d926ff4b45b1b772580dccc99561, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c] to archive 2024-11-24T02:44:05,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:06,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/188dfb627656437cab37a6eeef953bf0 2024-11-24T02:44:06,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/f2385b50a4a240b086d19483f230fcd0 2024-11-24T02:44:06,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0ed215187a964a29b7dd9eed009ae3d2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0ed215187a964a29b7dd9eed009ae3d2 2024-11-24T02:44:06,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/efea1927c98e40aeae71b917ec510d89 2024-11-24T02:44:06,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a1018f742cd04ba7a5cfda49e30a55fa 2024-11-24T02:44:06,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/947c885b8fb94a7f9264c935024fc87a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/947c885b8fb94a7f9264c935024fc87a 2024-11-24T02:44:06,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/c0abf9c6ab604885b8c55a5755ee9357 2024-11-24T02:44:06,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ee6c63e00df94eb0861b01eb38c19f0a 2024-11-24T02:44:06,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e91f2e8c84d84a13b9f97034ed4c408a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e91f2e8c84d84a13b9f97034ed4c408a 2024-11-24T02:44:06,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/65168ca4a9ef41d587b1b5199dac84ca 2024-11-24T02:44:06,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/bd85f839200d4b068fa3ac36e9a8bde6 2024-11-24T02:44:06,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/047f8f926a1d4c73963c9d0d52ce8031 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/047f8f926a1d4c73963c9d0d52ce8031 2024-11-24T02:44:06,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a6682f1f1e4d4d8cbd66b4723c2d8c52 2024-11-24T02:44:06,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/6abb0e6cb8ab47f2a13b81c99b0ce046 2024-11-24T02:44:06,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/e028fe416ed9436b83118728d27cacfc 2024-11-24T02:44:06,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/0a64ee927681468c8118690f2aa37ac0 2024-11-24T02:44:06,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/8874d926ff4b45b1b772580dccc99561 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/8874d926ff4b45b1b772580dccc99561 2024-11-24T02:44:06,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/fb6898934f304b238acc7276bf2dd838 2024-11-24T02:44:06,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/782318b53d2543fc88cf8c594f12b70d 2024-11-24T02:44:06,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/ea55470a446b4a32a699b01aa33a114c 2024-11-24T02:44:06,404 INFO [Thread-1525 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1525. 2024-11-24T02:44:06,404 DEBUG [Thread-1525 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:06,404 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,404 DEBUG [Thread-1525 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:06,411 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-24T02:44:06,411 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:06,411 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,412 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:06,421 INFO [Thread-1523 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1523. 2024-11-24T02:44:06,421 DEBUG [Thread-1523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:06,422 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,422 DEBUG [Thread-1523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-24T02:44:06,422 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3678 rows 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1235 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3705 rows 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1231 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3691 rows 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1236 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3707 rows 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-24T02:44:06,423 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3678 rows 2024-11-24T02:44:06,423 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:44:06,423 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:44:06,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:06,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:06,425 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:44:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:06,429 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416246429"}]},"ts":"1732416246429"} 2024-11-24T02:44:06,430 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:44:06,430 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:44:06,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:44:06,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, UNASSIGN}] 2024-11-24T02:44:06,433 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, UNASSIGN 2024-11-24T02:44:06,434 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:06,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, UNASSIGN because future has completed 2024-11-24T02:44:06,436 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:44:06,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:06,589 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:06,590 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:44:06,590 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing d4d30f9d55fca66379e8b4e679fdf056, disabling compactions & flushes 2024-11-24T02:44:06,590 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:06,590 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:06,590 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. after waiting 0 ms 2024-11-24T02:44:06,590 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:06,590 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing d4d30f9d55fca66379e8b4e679fdf056 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T02:44:06,591 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=A 2024-11-24T02:44:06,591 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:06,591 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=B 2024-11-24T02:44:06,591 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:06,592 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d4d30f9d55fca66379e8b4e679fdf056, store=C 2024-11-24T02:44:06,592 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:06,601 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240763370309544ab9af79544f2ebf4efd_d4d30f9d55fca66379e8b4e679fdf056 is 50, key is test_row_0/A:col10/1732416246409/Put/seqid=0 2024-11-24T02:44:06,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742247_1423 (size=12454) 2024-11-24T02:44:06,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:07,007 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:07,011 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240763370309544ab9af79544f2ebf4efd_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240763370309544ab9af79544f2ebf4efd_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:07,013 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/04c68068c08a49aeadaf7c5b08e904d6, store: [table=TestAcidGuarantees family=A region=d4d30f9d55fca66379e8b4e679fdf056] 2024-11-24T02:44:07,013 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/04c68068c08a49aeadaf7c5b08e904d6 is 175, key is test_row_0/A:col10/1732416246409/Put/seqid=0 2024-11-24T02:44:07,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742248_1424 (size=31255) 2024-11-24T02:44:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:07,418 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/04c68068c08a49aeadaf7c5b08e904d6 2024-11-24T02:44:07,431 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/5eb5a97d07a74741b5bcf6ddfb1520ab is 50, key is test_row_0/B:col10/1732416246409/Put/seqid=0 2024-11-24T02:44:07,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742249_1425 (size=12301) 2024-11-24T02:44:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:07,837 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/5eb5a97d07a74741b5bcf6ddfb1520ab 2024-11-24T02:44:07,849 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a052429b5b1f49858937608a344b6f2f is 50, key is test_row_0/C:col10/1732416246409/Put/seqid=0 2024-11-24T02:44:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742250_1426 (size=12301) 2024-11-24T02:44:08,253 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a052429b5b1f49858937608a344b6f2f 2024-11-24T02:44:08,262 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/A/04c68068c08a49aeadaf7c5b08e904d6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/04c68068c08a49aeadaf7c5b08e904d6 2024-11-24T02:44:08,267 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/04c68068c08a49aeadaf7c5b08e904d6, entries=150, sequenceid=300, filesize=30.5 K 2024-11-24T02:44:08,267 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/B/5eb5a97d07a74741b5bcf6ddfb1520ab as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/5eb5a97d07a74741b5bcf6ddfb1520ab 2024-11-24T02:44:08,270 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/5eb5a97d07a74741b5bcf6ddfb1520ab, entries=150, sequenceid=300, filesize=12.0 K 2024-11-24T02:44:08,271 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/.tmp/C/a052429b5b1f49858937608a344b6f2f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a052429b5b1f49858937608a344b6f2f 2024-11-24T02:44:08,273 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a052429b5b1f49858937608a344b6f2f, entries=150, sequenceid=300, filesize=12.0 K 2024-11-24T02:44:08,274 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d4d30f9d55fca66379e8b4e679fdf056 in 1684ms, sequenceid=300, compaction requested=true 2024-11-24T02:44:08,277 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-24T02:44:08,278 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. 2024-11-24T02:44:08,278 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for d4d30f9d55fca66379e8b4e679fdf056: Waiting for close lock at 1732416246590Running coprocessor pre-close hooks at 1732416246590Disabling compacts and flushes for region at 1732416246590Disabling writes for close at 1732416246590Obtaining lock to block concurrent updates at 1732416246590Preparing flush snapshotting stores in d4d30f9d55fca66379e8b4e679fdf056 at 1732416246590Finished memstore snapshotting TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732416246592 (+2 ms)Flushing stores of TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056. at 1732416246593 (+1 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/A: creating writer at 1732416246593Flushing mob file d4d30f9d55fca66379e8b4e679fdf056/A: appending metadata at 1732416246601 (+8 ms)Flushing mob file d4d30f9d55fca66379e8b4e679fdf056/A: closing flushed file at 1732416246601Flushing d4d30f9d55fca66379e8b4e679fdf056/A: appending metadata at 1732416247013 (+412 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/A: closing flushed file at 1732416247013Flushing d4d30f9d55fca66379e8b4e679fdf056/B: creating writer at 1732416247426 (+413 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/B: appending metadata at 1732416247431 (+5 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/B: closing flushed file at 1732416247431Flushing d4d30f9d55fca66379e8b4e679fdf056/C: creating writer at 1732416247846 (+415 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/C: appending metadata at 1732416247849 (+3 ms)Flushing d4d30f9d55fca66379e8b4e679fdf056/C: closing flushed file at 1732416247849Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b286b62: reopening flushed file at 1732416248261 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e152f27: reopening flushed file at 1732416248267 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c2eafd5: reopening flushed file at 1732416248270 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for d4d30f9d55fca66379e8b4e679fdf056 in 1684ms, sequenceid=300, compaction requested=true at 1732416248274 (+4 ms)Writing region close event to WAL at 1732416248275 (+1 ms)Running coprocessor post-close hooks at 1732416248277 (+2 ms)Closed at 1732416248277 2024-11-24T02:44:08,279 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,280 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=d4d30f9d55fca66379e8b4e679fdf056, regionState=CLOSED 2024-11-24T02:44:08,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:08,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T02:44:08,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure d4d30f9d55fca66379e8b4e679fdf056, server=7c69a60bd8f6,34251,1732416122955 in 1.8460 sec 2024-11-24T02:44:08,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-24T02:44:08,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d4d30f9d55fca66379e8b4e679fdf056, UNASSIGN in 1.8510 sec 2024-11-24T02:44:08,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T02:44:08,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8530 sec 2024-11-24T02:44:08,287 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416248287"}]},"ts":"1732416248287"} 2024-11-24T02:44:08,288 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:44:08,288 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:44:08,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-24T02:44:08,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:44:08,580 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:08,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:44:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,584 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:44:08,585 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,588 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,593 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits] 2024-11-24T02:44:08,597 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/04c68068c08a49aeadaf7c5b08e904d6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/04c68068c08a49aeadaf7c5b08e904d6 2024-11-24T02:44:08,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ad5e45044fc546819bbaafd8b113aefd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/ad5e45044fc546819bbaafd8b113aefd 2024-11-24T02:44:08,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c45e1452a5084fa6b26d8d94e458c91a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/A/c45e1452a5084fa6b26d8d94e458c91a 2024-11-24T02:44:08,604 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/0204a5806f9e4e87af59dc6d13aff033 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/0204a5806f9e4e87af59dc6d13aff033 2024-11-24T02:44:08,605 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/5eb5a97d07a74741b5bcf6ddfb1520ab to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/5eb5a97d07a74741b5bcf6ddfb1520ab 2024-11-24T02:44:08,607 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/e17802c72b29411086e1e09f8694b483 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/B/e17802c72b29411086e1e09f8694b483 2024-11-24T02:44:08,608 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/10ce90c22dbe462d956f989a9c8b7340 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/10ce90c22dbe462d956f989a9c8b7340 2024-11-24T02:44:08,609 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/973d6429df9849aea812429b70f88cc2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/973d6429df9849aea812429b70f88cc2 2024-11-24T02:44:08,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a052429b5b1f49858937608a344b6f2f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/C/a052429b5b1f49858937608a344b6f2f 2024-11-24T02:44:08,611 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits/303.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056/recovered.edits/303.seqid 2024-11-24T02:44:08,612 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,612 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:44:08,612 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:44:08,612 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:44:08,614 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240763370309544ab9af79544f2ebf4efd_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240763370309544ab9af79544f2ebf4efd_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,615 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431914b78495344858d155f6fb8a47589_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431914b78495344858d155f6fb8a47589_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,616 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c3aa89b83a54b3292fb6b456a870628_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c3aa89b83a54b3292fb6b456a870628_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,616 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112440d04eb911b846188b37b23ebdcf3954_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112440d04eb911b846188b37b23ebdcf3954_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,617 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bd1b8b993be4c2ead8445cc9cb30d3d_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bd1b8b993be4c2ead8445cc9cb30d3d_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,618 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124601dc6959da14865a3b90b23e94ee148_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124601dc6959da14865a3b90b23e94ee148_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,619 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469980737d3424010b398e8cf408e024f_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469980737d3424010b398e8cf408e024f_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,619 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247db3c3e5642a434d91071cdc932d35d5_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247db3c3e5642a434d91071cdc932d35d5_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,620 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480a0864ed23741db95e6d2037e617b8b_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480a0864ed23741db95e6d2037e617b8b_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,620 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b74b9cecf2b9410eb499de9cd190f16c_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b74b9cecf2b9410eb499de9cd190f16c_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,621 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2cce12e0567475dab024c902a6b8285_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2cce12e0567475dab024c902a6b8285_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,622 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c35dc8762cbd470ba3df83f9cc3d9659_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c35dc8762cbd470ba3df83f9cc3d9659_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,623 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccc3f9afd17145eaba5626ba4326889f_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ccc3f9afd17145eaba5626ba4326889f_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,624 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dabd43694c77474f9f3987242551f5cb_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dabd43694c77474f9f3987242551f5cb_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,624 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de203f0b2fbd4321ac535437bd3cf112_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124de203f0b2fbd4321ac535437bd3cf112_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,625 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1305cf7fe704c20a2f21814246b9ad1_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e1305cf7fe704c20a2f21814246b9ad1_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,626 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f35e5a1a4e9a4d4c9bf667cf96858744_d4d30f9d55fca66379e8b4e679fdf056 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f35e5a1a4e9a4d4c9bf667cf96858744_d4d30f9d55fca66379e8b4e679fdf056 2024-11-24T02:44:08,627 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:44:08,629 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,631 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:44:08,632 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:44:08,633 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,633 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:44:08,633 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416248633"}]},"ts":"9223372036854775807"} 2024-11-24T02:44:08,635 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:44:08,635 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d4d30f9d55fca66379e8b4e679fdf056, NAME => 'TestAcidGuarantees,,1732416218575.d4d30f9d55fca66379e8b4e679fdf056.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:44:08,635 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:44:08,635 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416248635"}]},"ts":"9223372036854775807"} 2024-11-24T02:44:08,636 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:44:08,636 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-24T02:44:08,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:44:08,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:44:08,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:08,703 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_693650994_22 at /127.0.0.1:45344 [Waiting for operation #115] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_693650994_22 at /127.0.0.1:53340 [Waiting for operation #592] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1130878907_22 at /127.0.0.1:45302 [Waiting for operation #136] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=10074 (was 10103) 2024-11-24T02:44:08,713 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=10074 2024-11-24T02:44:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:44:08,714 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:44:08,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:08,717 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:44:08,717 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:08,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-24T02:44:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:08,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:44:08,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742251_1427 (size=1061) 2024-11-24T02:44:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:09,130 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:44:09,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742252_1428 (size=53) 2024-11-24T02:44:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:09,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:44:09,542 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cfb4c06d27130e960c1fcd48fcbed8af, disabling compactions & flushes 2024-11-24T02:44:09,542 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. after waiting 0 ms 2024-11-24T02:44:09,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,543 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfb4c06d27130e960c1fcd48fcbed8af: Waiting for close lock at 1732416249542Disabling compacts and flushes for region at 1732416249542Disabling writes for close at 1732416249543 (+1 ms)Writing region close event to WAL at 1732416249543Closed at 1732416249543 2024-11-24T02:44:09,545 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:44:09,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416249545"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416249545"}]},"ts":"1732416249545"} 2024-11-24T02:44:09,548 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:44:09,549 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:44:09,549 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416249549"}]},"ts":"1732416249549"} 2024-11-24T02:44:09,551 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:44:09,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, ASSIGN}] 2024-11-24T02:44:09,553 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, ASSIGN 2024-11-24T02:44:09,554 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:44:09,705 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=cfb4c06d27130e960c1fcd48fcbed8af, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:09,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, ASSIGN because future has completed 2024-11-24T02:44:09,708 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:09,868 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,868 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:44:09,870 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,870 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:44:09,870 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,870 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,872 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,873 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:09,874 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb4c06d27130e960c1fcd48fcbed8af columnFamilyName A 2024-11-24T02:44:09,874 DEBUG [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:09,874 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(327): Store=cfb4c06d27130e960c1fcd48fcbed8af/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:09,875 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,876 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:09,876 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb4c06d27130e960c1fcd48fcbed8af columnFamilyName B 2024-11-24T02:44:09,876 DEBUG [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:09,877 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(327): Store=cfb4c06d27130e960c1fcd48fcbed8af/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:09,877 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,878 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:09,878 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfb4c06d27130e960c1fcd48fcbed8af columnFamilyName C 2024-11-24T02:44:09,879 DEBUG [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:09,879 INFO [StoreOpener-cfb4c06d27130e960c1fcd48fcbed8af-1 {}] regionserver.HStore(327): Store=cfb4c06d27130e960c1fcd48fcbed8af/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:09,879 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,879 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,880 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,881 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,882 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,882 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,883 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:44:09,884 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,886 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:44:09,886 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened cfb4c06d27130e960c1fcd48fcbed8af; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60271559, jitterRate=-0.10188378393650055}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:44:09,886 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:09,887 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for cfb4c06d27130e960c1fcd48fcbed8af: Running coprocessor pre-open hook at 1732416249870Writing region info on filesystem at 1732416249870Initializing all the Stores at 1732416249871 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416249871Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416249872 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416249872Cleaning up temporary data from old regions at 1732416249882 (+10 ms)Running coprocessor post-open hooks at 1732416249886 (+4 ms)Region opened successfully at 1732416249887 (+1 ms) 2024-11-24T02:44:09,887 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., pid=114, masterSystemTime=1732416249861 2024-11-24T02:44:09,889 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,889 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:09,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=cfb4c06d27130e960c1fcd48fcbed8af, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:09,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:09,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T02:44:09,893 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 in 184 msec 2024-11-24T02:44:09,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-24T02:44:09,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, ASSIGN in 342 msec 2024-11-24T02:44:09,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:44:09,895 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416249895"}]},"ts":"1732416249895"} 2024-11-24T02:44:09,896 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:44:09,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:44:09,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-24T02:44:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-24T02:44:10,870 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:10,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a3a0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,875 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75fc8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,877 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8af9e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,878 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,879 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:44:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:44:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@205f94f4 2024-11-24T02:44:10,882 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:44:10,883 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:44:10,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a75b829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28623480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ce1a7c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,889 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17336170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,890 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cef9d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,891 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417a9424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,893 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6be9d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32833dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,895 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4304bdf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddd5e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7916b933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,899 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4576fdef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14abdf85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,902 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6602918a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,904 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee16b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,904 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66b4527c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,906 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,906 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,907 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bdb82f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,908 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f17f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0bbd77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:10,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:10,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f63659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:10,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443cf22b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:10,913 DEBUG [Thread-1946 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,913 DEBUG [Thread-1944 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,913 DEBUG [Thread-1935 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,913 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,914 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,914 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:10,914 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,914 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-24T02:44:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,916 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,918 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,919 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,920 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:10,920 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:10,920 DEBUG [Thread-1940 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,920 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:10,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2] 2024-11-24T02:44:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:10,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:10,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:10,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/d073ce92962744b2b1cd00aaae072244 is 50, key is test_row_0/A:col10/1732416250926/Put/seqid=0 2024-11-24T02:44:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416310950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:10,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416250943, max timestamp=1732416250957Num uniques 859; 2024-11-24T02:44:10,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:44:10,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416310954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:10,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416250943, max timestamp=1732416250957Num uniques 859; 2024-11-24T02:44:10,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732416250943, max timestamp=1732416250957Num uniques 859; 2024-11-24T02:44:10,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:44:10,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:10,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742253_1429 (size=9657) 2024-11-24T02:44:11,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416310990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416310996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416310997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:11,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416311069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416311069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416311109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416311109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416311109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:11,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416311289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416311290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416311319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416311319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416311319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/d073ce92962744b2b1cd00aaae072244 2024-11-24T02:44:11,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6a5439d6efb24597830d29d31a26f94f is 50, key is test_row_0/B:col10/1732416250926/Put/seqid=0 2024-11-24T02:44:11,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742254_1430 (size=9657) 2024-11-24T02:44:11,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416311609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416311610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416311629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416311629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416311629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:11,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6a5439d6efb24597830d29d31a26f94f 2024-11-24T02:44:11,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/48ac6e47fa934dbab92d26440823034d is 50, key is test_row_0/C:col10/1732416250926/Put/seqid=0 2024-11-24T02:44:11,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742255_1431 (size=9657) 2024-11-24T02:44:11,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/48ac6e47fa934dbab92d26440823034d 2024-11-24T02:44:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/d073ce92962744b2b1cd00aaae072244 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244 2024-11-24T02:44:11,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T02:44:11,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6a5439d6efb24597830d29d31a26f94f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f 2024-11-24T02:44:11,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T02:44:11,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/48ac6e47fa934dbab92d26440823034d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d 2024-11-24T02:44:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:11,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:11,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d, entries=100, sequenceid=12, filesize=9.4 K 2024-11-24T02:44:11,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cfb4c06d27130e960c1fcd48fcbed8af in 917ms, sequenceid=12, compaction requested=false 2024-11-24T02:44:11,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:11,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-24T02:44:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:11,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-24T02:44:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:11,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:11,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2580441f1a934fe582e4b6c345366e9c is 50, key is test_row_0/A:col10/1732416250995/Put/seqid=0 2024-11-24T02:44:12,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742256_1432 (size=12001) 2024-11-24T02:44:12,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2580441f1a934fe582e4b6c345366e9c 2024-11-24T02:44:12,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/fb8f24758a5a405db186ab0cdd4c7f95 is 50, key is test_row_0/B:col10/1732416250995/Put/seqid=0 2024-11-24T02:44:12,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742257_1433 (size=12001) 2024-11-24T02:44:12,016 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/fb8f24758a5a405db186ab0cdd4c7f95 2024-11-24T02:44:12,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/3f3669deff6f4e2bbf7facadddd6a79f is 50, key is test_row_0/C:col10/1732416250995/Put/seqid=0 2024-11-24T02:44:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742258_1434 (size=12001) 2024-11-24T02:44:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:12,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416312140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416312147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416312148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416312149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416312150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416312259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416312259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416312268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416312269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416312269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,426 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/3f3669deff6f4e2bbf7facadddd6a79f 2024-11-24T02:44:12,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2580441f1a934fe582e4b6c345366e9c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c 2024-11-24T02:44:12,432 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T02:44:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/fb8f24758a5a405db186ab0cdd4c7f95 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95 2024-11-24T02:44:12,435 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T02:44:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/3f3669deff6f4e2bbf7facadddd6a79f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f 2024-11-24T02:44:12,439 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T02:44:12,440 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=60.38 KB/61830 for cfb4c06d27130e960c1fcd48fcbed8af in 448ms, sequenceid=46, compaction requested=false 2024-11-24T02:44:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:12,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-24T02:44:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-24T02:44:12,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T02:44:12,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5200 sec 2024-11-24T02:44:12,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5290 sec 2024-11-24T02:44:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:12,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:12,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:12,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/c94f75af511641eca8767f67daa8067e is 50, key is test_row_0/A:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742259_1435 (size=14341) 2024-11-24T02:44:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416312507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416312508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416312515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416312519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416312519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416312630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416312839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416312839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416312840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416312840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:12,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416312840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:12,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:12,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/c94f75af511641eca8767f67daa8067e 2024-11-24T02:44:12,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/117f7280b82e4654aabc4922f380bdc2 is 50, key is test_row_0/B:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:12,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742260_1436 (size=12001) 2024-11-24T02:44:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T02:44:13,079 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:13,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-24T02:44:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:13,081 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:13,081 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:13,081 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416313149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416313149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416313158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416313159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416313159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:13,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T02:44:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/117f7280b82e4654aabc4922f380bdc2 2024-11-24T02:44:13,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/cca457bfc7f146a3a5fc7661ba7e20f1 is 50, key is test_row_0/C:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:13,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742261_1437 (size=12001) 2024-11-24T02:44:13,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T02:44:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:13,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T02:44:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416313668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416313669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416313679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416313679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416313679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:13,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:13,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T02:44:13,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:13,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:13,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/cca457bfc7f146a3a5fc7661ba7e20f1 2024-11-24T02:44:13,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/c94f75af511641eca8767f67daa8067e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e 2024-11-24T02:44:13,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e, entries=200, sequenceid=59, filesize=14.0 K 2024-11-24T02:44:13,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/117f7280b82e4654aabc4922f380bdc2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2 2024-11-24T02:44:13,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2, entries=150, sequenceid=59, filesize=11.7 K 2024-11-24T02:44:13,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/cca457bfc7f146a3a5fc7661ba7e20f1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1 2024-11-24T02:44:13,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1, entries=150, sequenceid=59, filesize=11.7 K 2024-11-24T02:44:13,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cfb4c06d27130e960c1fcd48fcbed8af in 1242ms, sequenceid=59, compaction requested=true 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:13,725 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:13,725 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:13,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:13,726 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,726 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,726 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.2 K 2024-11-24T02:44:13,726 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=32.9 K 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d073ce92962744b2b1cd00aaae072244, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416250926 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a5439d6efb24597830d29d31a26f94f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416250926 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fb8f24758a5a405db186ab0cdd4c7f95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732416250984 2024-11-24T02:44:13,726 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2580441f1a934fe582e4b6c345366e9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732416250984 2024-11-24T02:44:13,727 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 117f7280b82e4654aabc4922f380bdc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252148 2024-11-24T02:44:13,727 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94f75af511641eca8767f67daa8067e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252147 2024-11-24T02:44:13,735 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:13,735 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/32468a1274944049ac6f5853be88a8b9 is 50, key is test_row_0/B:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:13,742 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:13,743 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/20faa751e15d404a9c5e0ee9c83d1602 is 50, key is test_row_0/A:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:13,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742262_1438 (size=12104) 2024-11-24T02:44:13,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742263_1439 (size=12104) 2024-11-24T02:44:13,754 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/32468a1274944049ac6f5853be88a8b9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/32468a1274944049ac6f5853be88a8b9 2024-11-24T02:44:13,761 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 32468a1274944049ac6f5853be88a8b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:13,761 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:13,761 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416253725; duration=0sec 2024-11-24T02:44:13,761 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:13,761 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:13,761 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:13,762 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:13,762 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:13,762 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,762 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=32.9 K 2024-11-24T02:44:13,763 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 48ac6e47fa934dbab92d26440823034d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416250926 2024-11-24T02:44:13,764 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f3669deff6f4e2bbf7facadddd6a79f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732416250984 2024-11-24T02:44:13,764 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting cca457bfc7f146a3a5fc7661ba7e20f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252148 2024-11-24T02:44:13,769 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:13,770 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/cc982fac096b4ce5ad3f4366f2fc537b is 50, key is test_row_0/C:col10/1732416252148/Put/seqid=0 2024-11-24T02:44:13,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742264_1440 (size=12104) 2024-11-24T02:44:13,794 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:44:13,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:13,843 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:13,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/790a927c33da4bf59611c90f640dcc33 is 50, key is test_row_0/A:col10/1732416252517/Put/seqid=0 2024-11-24T02:44:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742265_1441 (size=12001) 2024-11-24T02:44:14,156 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/20faa751e15d404a9c5e0ee9c83d1602 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/20faa751e15d404a9c5e0ee9c83d1602 2024-11-24T02:44:14,159 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 20faa751e15d404a9c5e0ee9c83d1602(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:14,159 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:14,159 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416253725; duration=0sec 2024-11-24T02:44:14,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:14,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:14,177 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/cc982fac096b4ce5ad3f4366f2fc537b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cc982fac096b4ce5ad3f4366f2fc537b 2024-11-24T02:44:14,180 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into cc982fac096b4ce5ad3f4366f2fc537b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:14,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:14,181 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416253725; duration=0sec 2024-11-24T02:44:14,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:14,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:14,251 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/790a927c33da4bf59611c90f640dcc33 2024-11-24T02:44:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/015a143c5d394532b854ac06fa63a7a4 is 50, key is test_row_0/B:col10/1732416252517/Put/seqid=0 2024-11-24T02:44:14,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742266_1442 (size=12001) 2024-11-24T02:44:14,662 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/015a143c5d394532b854ac06fa63a7a4 2024-11-24T02:44:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/96a6d2c18b904ae3a576433504ad0270 is 50, key is test_row_0/C:col10/1732416252517/Put/seqid=0 2024-11-24T02:44:14,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742267_1443 (size=12001) 2024-11-24T02:44:14,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:14,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416314698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416314699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416314699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,707 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416314705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416314707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416314808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416314819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416314819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416314819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416315018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416315029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,071 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/96a6d2c18b904ae3a576433504ad0270 2024-11-24T02:44:15,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/790a927c33da4bf59611c90f640dcc33 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33 2024-11-24T02:44:15,078 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:44:15,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/015a143c5d394532b854ac06fa63a7a4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4 2024-11-24T02:44:15,082 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:44:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/96a6d2c18b904ae3a576433504ad0270 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270 2024-11-24T02:44:15,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:44:15,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cfb4c06d27130e960c1fcd48fcbed8af in 1243ms, sequenceid=79, compaction requested=false 2024-11-24T02:44:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-24T02:44:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-24T02:44:15,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-24T02:44:15,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0070 sec 2024-11-24T02:44:15,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.0110 sec 2024-11-24T02:44:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:44:15,229 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:15,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-24T02:44:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:15,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:15,231 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:15,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:15,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:15,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/39f47cc2fa60428c9ea0781d8da895c7 is 50, key is test_row_0/A:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742268_1444 (size=14341) 2024-11-24T02:44:15,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416315366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416315367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T02:44:15,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416315378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416315378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416315489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416315489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416315498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416315498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T02:44:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:15,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T02:44:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:15,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416315699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416315708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416315708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416315708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:15,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/39f47cc2fa60428c9ea0781d8da895c7 2024-11-24T02:44:15,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/da5582eebc934a23b7a05a6223fef83a is 50, key is test_row_0/B:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742269_1445 (size=12001) 2024-11-24T02:44:15,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/da5582eebc934a23b7a05a6223fef83a 2024-11-24T02:44:15,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/5255e53bb567473bb5ad2fb6ce14b9c7 is 50, key is test_row_0/C:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742270_1446 (size=12001) 2024-11-24T02:44:15,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/5255e53bb567473bb5ad2fb6ce14b9c7 2024-11-24T02:44:15,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/39f47cc2fa60428c9ea0781d8da895c7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7 2024-11-24T02:44:15,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7, entries=200, sequenceid=95, filesize=14.0 K 2024-11-24T02:44:15,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/da5582eebc934a23b7a05a6223fef83a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a 2024-11-24T02:44:15,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T02:44:15,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/5255e53bb567473bb5ad2fb6ce14b9c7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7 2024-11-24T02:44:15,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T02:44:15,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cfb4c06d27130e960c1fcd48fcbed8af in 443ms, sequenceid=95, compaction requested=true 2024-11-24T02:44:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:15,778 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:15,779 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:15,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:15,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:15,779 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:15,779 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:15,780 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,780 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,780 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/20faa751e15d404a9c5e0ee9c83d1602, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=37.5 K 2024-11-24T02:44:15,780 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/32468a1274944049ac6f5853be88a8b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.3 K 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20faa751e15d404a9c5e0ee9c83d1602, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252148 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 32468a1274944049ac6f5853be88a8b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252148 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 015a143c5d394532b854ac06fa63a7a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416252507 2024-11-24T02:44:15,780 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790a927c33da4bf59611c90f640dcc33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416252507 2024-11-24T02:44:15,781 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39f47cc2fa60428c9ea0781d8da895c7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:15,781 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting da5582eebc934a23b7a05a6223fef83a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:15,786 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:15,786 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/3a86d90678a84973958efa5d1bfef47b is 50, key is test_row_0/B:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,786 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:15,787 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/97532476f7834de79a03ca3e80076ae7 is 50, key is test_row_0/A:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742271_1447 (size=12207) 2024-11-24T02:44:15,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742272_1448 (size=12207) 2024-11-24T02:44:15,794 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/97532476f7834de79a03ca3e80076ae7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/97532476f7834de79a03ca3e80076ae7 2024-11-24T02:44:15,798 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 97532476f7834de79a03ca3e80076ae7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:15,798 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:15,798 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416255778; duration=0sec 2024-11-24T02:44:15,798 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:15,798 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:15,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:15,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:15,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:15,799 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,799 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cc982fac096b4ce5ad3f4366f2fc537b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.3 K 2024-11-24T02:44:15,799 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc982fac096b4ce5ad3f4366f2fc537b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732416252148 2024-11-24T02:44:15,800 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a6d2c18b904ae3a576433504ad0270, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416252507 2024-11-24T02:44:15,800 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5255e53bb567473bb5ad2fb6ce14b9c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:15,806 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:15,806 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/4f4bcca38a2b4309a0cc3e76f5d302ee is 50, key is test_row_0/C:col10/1732416255334/Put/seqid=0 2024-11-24T02:44:15,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742273_1449 (size=12207) 2024-11-24T02:44:15,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-24T02:44:15,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:15,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:15,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3727dd87c0ff4ebe949463c8badc44f7 is 50, key is test_row_0/A:col10/1732416255375/Put/seqid=0 2024-11-24T02:44:15,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742274_1450 (size=12001) 2024-11-24T02:44:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:16,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416316038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416316038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416316039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416316041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416316148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416316159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416316159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416316159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,195 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/3a86d90678a84973958efa5d1bfef47b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3a86d90678a84973958efa5d1bfef47b 2024-11-24T02:44:16,198 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 3a86d90678a84973958efa5d1bfef47b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:16,198 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:16,198 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416255778; duration=0sec 2024-11-24T02:44:16,199 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:16,199 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:16,225 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/4f4bcca38a2b4309a0cc3e76f5d302ee as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4f4bcca38a2b4309a0cc3e76f5d302ee 2024-11-24T02:44:16,228 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into 4f4bcca38a2b4309a0cc3e76f5d302ee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:16,228 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:16,228 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416255778; duration=0sec 2024-11-24T02:44:16,228 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:16,229 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:16,249 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3727dd87c0ff4ebe949463c8badc44f7 2024-11-24T02:44:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b55ee07207014672a71941028f8f0c33 is 50, key is test_row_0/B:col10/1732416255375/Put/seqid=0 2024-11-24T02:44:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742275_1451 (size=12001) 2024-11-24T02:44:16,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416316359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416316369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416316379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416316379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b55ee07207014672a71941028f8f0c33 2024-11-24T02:44:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/201621d1d13341fa9eee481a8ec2fd57 is 50, key is test_row_0/C:col10/1732416255375/Put/seqid=0 2024-11-24T02:44:16,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742276_1452 (size=12001) 2024-11-24T02:44:16,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416316669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416316678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416316689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416316699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416316719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:16,727 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,067 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/201621d1d13341fa9eee481a8ec2fd57 2024-11-24T02:44:17,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3727dd87c0ff4ebe949463c8badc44f7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7 2024-11-24T02:44:17,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:44:17,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b55ee07207014672a71941028f8f0c33 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33 2024-11-24T02:44:17,079 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:44:17,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/201621d1d13341fa9eee481a8ec2fd57 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57 2024-11-24T02:44:17,083 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:44:17,083 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cfb4c06d27130e960c1fcd48fcbed8af in 1242ms, sequenceid=114, compaction requested=false 2024-11-24T02:44:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-24T02:44:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-24T02:44:17,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T02:44:17,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8530 sec 2024-11-24T02:44:17,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8570 sec 2024-11-24T02:44:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:17,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:44:17,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:17,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:17,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:17,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:17,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:17,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:17,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/13f72177977e45e784439a3a913de592 is 50, key is test_row_0/A:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742277_1453 (size=14391) 2024-11-24T02:44:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416317223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416317229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416317230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416317232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416317339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416317348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416317348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416317349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T02:44:17,379 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:17,379 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T02:44:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:17,381 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:17,382 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:17,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:17,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:44:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416317548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416317558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416317559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416317559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/13f72177977e45e784439a3a913de592 2024-11-24T02:44:17,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/231a618dad2d4e2f9d2c19790a5c8b0a is 50, key is test_row_0/B:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742278_1454 (size=12051) 2024-11-24T02:44:17,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/231a618dad2d4e2f9d2c19790a5c8b0a 2024-11-24T02:44:17,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/965eefb7bf364c92a04230a173fb5e7d is 50, key is test_row_0/C:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:17,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742279_1455 (size=12051) 2024-11-24T02:44:17,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:44:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:17,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:44:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416317859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416317879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416317879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416317880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:17,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:17,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:44:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:17,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:17,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:18,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/965eefb7bf364c92a04230a173fb5e7d 2024-11-24T02:44:18,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/13f72177977e45e784439a3a913de592 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592 2024-11-24T02:44:18,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592, entries=200, sequenceid=131, filesize=14.1 K 2024-11-24T02:44:18,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/231a618dad2d4e2f9d2c19790a5c8b0a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a 2024-11-24T02:44:18,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T02:44:18,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/965eefb7bf364c92a04230a173fb5e7d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d 2024-11-24T02:44:18,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T02:44:18,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cfb4c06d27130e960c1fcd48fcbed8af in 850ms, sequenceid=131, compaction requested=true 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:18,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:18,035 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:18,035 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:18,035 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:18,035 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/97532476f7834de79a03ca3e80076ae7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=37.7 K 2024-11-24T02:44:18,035 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3a86d90678a84973958efa5d1bfef47b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.4 K 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a86d90678a84973958efa5d1bfef47b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97532476f7834de79a03ca3e80076ae7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3727dd87c0ff4ebe949463c8badc44f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416255363 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b55ee07207014672a71941028f8f0c33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416255363 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f72177977e45e784439a3a913de592, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:18,036 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 231a618dad2d4e2f9d2c19790a5c8b0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:18,053 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:18,053 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/55e023bcc6404bc4a2a1c4b7792c987e is 50, key is test_row_0/A:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:18,056 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:18,056 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/fac5316f55d0474cadfe917d0aa56932 is 50, key is test_row_0/B:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:18,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742280_1456 (size=12359) 2024-11-24T02:44:18,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742281_1457 (size=12359) 2024-11-24T02:44:18,071 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/fac5316f55d0474cadfe917d0aa56932 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fac5316f55d0474cadfe917d0aa56932 2024-11-24T02:44:18,075 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into fac5316f55d0474cadfe917d0aa56932(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:18,075 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:18,075 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416258035; duration=0sec 2024-11-24T02:44:18,075 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:18,075 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:18,075 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:18,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:18,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:18,079 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:18,079 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4f4bcca38a2b4309a0cc3e76f5d302ee, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.4 K 2024-11-24T02:44:18,079 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4bcca38a2b4309a0cc3e76f5d302ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732416254694 2024-11-24T02:44:18,079 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 201621d1d13341fa9eee481a8ec2fd57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416255363 2024-11-24T02:44:18,079 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 965eefb7bf364c92a04230a173fb5e7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:18,088 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:18,088 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/1766617e9eac4a88a6ecf8fd3e90d40f is 50, key is test_row_0/C:col10/1732416256036/Put/seqid=0 2024-11-24T02:44:18,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742282_1458 (size=12359) 2024-11-24T02:44:18,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:44:18,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/bddbe55bd78c483da2b30919c3a5fcc4 is 50, key is test_row_0/A:col10/1732416257211/Put/seqid=0 2024-11-24T02:44:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742283_1459 (size=12151) 2024-11-24T02:44:18,151 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/bddbe55bd78c483da2b30919c3a5fcc4 2024-11-24T02:44:18,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/1e7e757f84474c8189b24d831fb02a5a is 50, key is test_row_0/B:col10/1732416257211/Put/seqid=0 2024-11-24T02:44:18,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742284_1460 (size=12151) 2024-11-24T02:44:18,161 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/1e7e757f84474c8189b24d831fb02a5a 2024-11-24T02:44:18,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7b2b91a1efb7455db9a0b5d6d3ddd02d is 50, key is test_row_0/C:col10/1732416257211/Put/seqid=0 2024-11-24T02:44:18,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742285_1461 (size=12151) 2024-11-24T02:44:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:18,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416318408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416318412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416318420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416318421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,470 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/55e023bcc6404bc4a2a1c4b7792c987e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/55e023bcc6404bc4a2a1c4b7792c987e 2024-11-24T02:44:18,473 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 55e023bcc6404bc4a2a1c4b7792c987e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:18,473 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:18,473 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416258035; duration=0sec 2024-11-24T02:44:18,473 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:18,473 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:18,495 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/1766617e9eac4a88a6ecf8fd3e90d40f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/1766617e9eac4a88a6ecf8fd3e90d40f 2024-11-24T02:44:18,498 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into 1766617e9eac4a88a6ecf8fd3e90d40f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:18,498 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:18,498 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416258035; duration=0sec 2024-11-24T02:44:18,498 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:18,498 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416318529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416318529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416318529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416318539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7b2b91a1efb7455db9a0b5d6d3ddd02d 2024-11-24T02:44:18,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/bddbe55bd78c483da2b30919c3a5fcc4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4 2024-11-24T02:44:18,576 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T02:44:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/1e7e757f84474c8189b24d831fb02a5a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a 2024-11-24T02:44:18,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T02:44:18,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7b2b91a1efb7455db9a0b5d6d3ddd02d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d 2024-11-24T02:44:18,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d, entries=150, sequenceid=149, filesize=11.9 K 2024-11-24T02:44:18,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cfb4c06d27130e960c1fcd48fcbed8af in 441ms, sequenceid=149, compaction requested=false 2024-11-24T02:44:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T02:44:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T02:44:18,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T02:44:18,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2020 sec 2024-11-24T02:44:18,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.2080 sec 2024-11-24T02:44:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:18,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:18,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2cd24d46b9924451896aaeef5d4f1698 is 50, key is test_row_0/A:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:18,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742286_1462 (size=16931) 2024-11-24T02:44:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416318769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416318770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416318770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416318773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416318879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416318889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416318889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416318889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:18,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416319099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416319099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416319099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416319099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2cd24d46b9924451896aaeef5d4f1698 2024-11-24T02:44:19,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eec9816f50b74efba53b2ff5a184ec80 is 50, key is test_row_0/B:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:19,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742287_1463 (size=12151) 2024-11-24T02:44:19,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416319408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416319409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416319409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416319409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:44:19,539 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:19,540 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T02:44:19,541 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:19,542 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:19,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:19,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eec9816f50b74efba53b2ff5a184ec80 2024-11-24T02:44:19,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/6e1fa747fab6418babe38ce5719b86a4 is 50, key is test_row_0/C:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742288_1464 (size=12151) 2024-11-24T02:44:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:19,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T02:44:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T02:44:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416319919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416319929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416319929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416319929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:19,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/6e1fa747fab6418babe38ce5719b86a4 2024-11-24T02:44:19,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2cd24d46b9924451896aaeef5d4f1698 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698 2024-11-24T02:44:19,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698, entries=250, sequenceid=168, filesize=16.5 K 2024-11-24T02:44:19,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eec9816f50b74efba53b2ff5a184ec80 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80 2024-11-24T02:44:19,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T02:44:19,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/6e1fa747fab6418babe38ce5719b86a4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4 2024-11-24T02:44:19,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4, entries=150, sequenceid=168, filesize=11.9 K 2024-11-24T02:44:19,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for cfb4c06d27130e960c1fcd48fcbed8af in 1250ms, sequenceid=168, compaction requested=true 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:19,994 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:19,994 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:19,995 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:19,995 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:19,995 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:19,995 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:19,995 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:19,995 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,995 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:19,995 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/55e023bcc6404bc4a2a1c4b7792c987e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=40.5 K 2024-11-24T02:44:19,995 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fac5316f55d0474cadfe917d0aa56932, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.8 K 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e023bcc6404bc4a2a1c4b7792c987e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fac5316f55d0474cadfe917d0aa56932, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddbe55bd78c483da2b30919c3a5fcc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732416257211 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e7e757f84474c8189b24d831fb02a5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732416257211 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd24d46b9924451896aaeef5d4f1698, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:19,996 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting eec9816f50b74efba53b2ff5a184ec80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:20,002 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:20,002 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/1f97f29626564aa592f14134c5bda5e4 is 50, key is test_row_0/A:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:20,002 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:20,003 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/5aa06c93fcfc42c5a6006e4a7295d2ca is 50, key is test_row_0/B:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:20,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T02:44:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:20,003 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T02:44:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742289_1465 (size=12561) 2024-11-24T02:44:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/82065058804a4b6bbbf7a72ad36fcb43 is 50, key is test_row_0/A:col10/1732416258766/Put/seqid=0 2024-11-24T02:44:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742290_1466 (size=12561) 2024-11-24T02:44:20,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742291_1467 (size=12151) 2024-11-24T02:44:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:20,411 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/1f97f29626564aa592f14134c5bda5e4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/1f97f29626564aa592f14134c5bda5e4 2024-11-24T02:44:20,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/82065058804a4b6bbbf7a72ad36fcb43 2024-11-24T02:44:20,415 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/5aa06c93fcfc42c5a6006e4a7295d2ca as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5aa06c93fcfc42c5a6006e4a7295d2ca 2024-11-24T02:44:20,415 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 1f97f29626564aa592f14134c5bda5e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:20,416 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416259994; duration=0sec 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:20,416 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:20,416 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:20,416 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/1766617e9eac4a88a6ecf8fd3e90d40f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=35.8 K 2024-11-24T02:44:20,417 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1766617e9eac4a88a6ecf8fd3e90d40f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416256036 2024-11-24T02:44:20,417 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2b91a1efb7455db9a0b5d6d3ddd02d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732416257211 2024-11-24T02:44:20,417 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1fa747fab6418babe38ce5719b86a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:20,420 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 5aa06c93fcfc42c5a6006e4a7295d2ca(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:20,420 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:20,420 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416259994; duration=0sec 2024-11-24T02:44:20,420 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:20,420 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/e13ac1f082e54100a388d0926370341b is 50, key is test_row_0/B:col10/1732416258766/Put/seqid=0 2024-11-24T02:44:20,424 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:20,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742292_1468 (size=12151) 2024-11-24T02:44:20,425 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/b11e98599e0446bba127bd332b6d0a34 is 50, key is test_row_0/C:col10/1732416258411/Put/seqid=0 2024-11-24T02:44:20,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742293_1469 (size=12561) 2024-11-24T02:44:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:20,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:20,825 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/e13ac1f082e54100a388d0926370341b 2024-11-24T02:44:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416320821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:20,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/4cd488e22ed4408c936eb8e1af02ab29 is 50, key is test_row_0/C:col10/1732416258766/Put/seqid=0 2024-11-24T02:44:20,832 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/b11e98599e0446bba127bd332b6d0a34 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/b11e98599e0446bba127bd332b6d0a34 2024-11-24T02:44:20,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742294_1470 (size=12151) 2024-11-24T02:44:20,834 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/4cd488e22ed4408c936eb8e1af02ab29 2024-11-24T02:44:20,836 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into b11e98599e0446bba127bd332b6d0a34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:20,836 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:20,836 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416259994; duration=0sec 2024-11-24T02:44:20,836 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:20,836 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:20,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/82065058804a4b6bbbf7a72ad36fcb43 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43 2024-11-24T02:44:20,841 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T02:44:20,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/e13ac1f082e54100a388d0926370341b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b 2024-11-24T02:44:20,844 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T02:44:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/4cd488e22ed4408c936eb8e1af02ab29 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29 2024-11-24T02:44:20,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29, entries=150, sequenceid=183, filesize=11.9 K 2024-11-24T02:44:20,848 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for cfb4c06d27130e960c1fcd48fcbed8af in 845ms, sequenceid=183, compaction requested=false 2024-11-24T02:44:20,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:20,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:20,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T02:44:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T02:44:20,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T02:44:20,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3080 sec 2024-11-24T02:44:20,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3120 sec 2024-11-24T02:44:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:20,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:20,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:20,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/b3ccf15146c44532a60a59da9720eda7 is 50, key is test_row_0/A:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:20,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742295_1471 (size=14541) 2024-11-24T02:44:20,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416320959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416320986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416320986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416320986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416320987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416321089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416321109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416321299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416321319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416321329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416321329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416321329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/b3ccf15146c44532a60a59da9720eda7 2024-11-24T02:44:21,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/70d6e43f708a45d49ad5c16f1c3b944b is 50, key is test_row_0/B:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:21,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742296_1472 (size=12151) 2024-11-24T02:44:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416321609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416321629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416321639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416321648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416321648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:44:21,689 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:21,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T02:44:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:21,691 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:21,692 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:21,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:21,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/70d6e43f708a45d49ad5c16f1c3b944b 2024-11-24T02:44:21,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f51faa82851b422cafef5c4a108c1ba0 is 50, key is test_row_0/C:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:21,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742297_1473 (size=12151) 2024-11-24T02:44:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:21,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T02:44:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:21,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:21,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T02:44:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:21,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:22,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416322129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:22,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T02:44:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416322148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416322149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:22,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416322159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:22,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f51faa82851b422cafef5c4a108c1ba0 2024-11-24T02:44:22,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/b3ccf15146c44532a60a59da9720eda7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7 2024-11-24T02:44:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416322169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:22,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7, entries=200, sequenceid=204, filesize=14.2 K 2024-11-24T02:44:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:22,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/70d6e43f708a45d49ad5c16f1c3b944b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b 2024-11-24T02:44:22,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T02:44:22,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f51faa82851b422cafef5c4a108c1ba0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0 2024-11-24T02:44:22,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T02:44:22,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for cfb4c06d27130e960c1fcd48fcbed8af in 1247ms, sequenceid=204, compaction requested=true 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:22,180 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:22,180 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:22,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:22,181 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,181 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,181 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5aa06c93fcfc42c5a6006e4a7295d2ca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.0 K 2024-11-24T02:44:22,181 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/1f97f29626564aa592f14134c5bda5e4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=38.3 K 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f97f29626564aa592f14134c5bda5e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aa06c93fcfc42c5a6006e4a7295d2ca, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82065058804a4b6bbbf7a72ad36fcb43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732416258766 2024-11-24T02:44:22,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e13ac1f082e54100a388d0926370341b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732416258766 2024-11-24T02:44:22,182 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ccf15146c44532a60a59da9720eda7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:22,182 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 70d6e43f708a45d49ad5c16f1c3b944b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:22,188 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:22,189 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/3336ec8e3b184ead8f93cf35e623bacf is 50, key is test_row_0/B:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:22,194 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:22,195 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/0a1e707d8f034920a0b7fb38c4dc0499 is 50, key is test_row_0/A:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:22,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742298_1474 (size=12663) 2024-11-24T02:44:22,203 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/3336ec8e3b184ead8f93cf35e623bacf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3336ec8e3b184ead8f93cf35e623bacf 2024-11-24T02:44:22,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742299_1475 (size=12663) 2024-11-24T02:44:22,207 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 3336ec8e3b184ead8f93cf35e623bacf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:22,207 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:22,207 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416262180; duration=0sec 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/0a1e707d8f034920a0b7fb38c4dc0499 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/0a1e707d8f034920a0b7fb38c4dc0499 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:22,208 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:22,208 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,208 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/b11e98599e0446bba127bd332b6d0a34, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.0 K 2024-11-24T02:44:22,209 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b11e98599e0446bba127bd332b6d0a34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732416258411 2024-11-24T02:44:22,209 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd488e22ed4408c936eb8e1af02ab29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732416258766 2024-11-24T02:44:22,211 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f51faa82851b422cafef5c4a108c1ba0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:22,212 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 0a1e707d8f034920a0b7fb38c4dc0499(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:22,212 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:22,212 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416262180; duration=0sec 2024-11-24T02:44:22,212 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:22,212 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:22,217 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:22,218 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/686c832f519d4e2aaf9319b4f8e08b03 is 50, key is test_row_0/C:col10/1732416260820/Put/seqid=0 2024-11-24T02:44:22,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742300_1476 (size=12663) 2024-11-24T02:44:22,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T02:44:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:22,302 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:22,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/f2028bf6416b4bdea820b01c44a8bd5a is 50, key is test_row_0/A:col10/1732416260986/Put/seqid=0 2024-11-24T02:44:22,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742301_1477 (size=12151) 2024-11-24T02:44:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:22,624 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/686c832f519d4e2aaf9319b4f8e08b03 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/686c832f519d4e2aaf9319b4f8e08b03 2024-11-24T02:44:22,628 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into 686c832f519d4e2aaf9319b4f8e08b03(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:22,628 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:22,628 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416262180; duration=0sec 2024-11-24T02:44:22,628 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:22,628 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:22,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/f2028bf6416b4bdea820b01c44a8bd5a 2024-11-24T02:44:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b2931b404d0e4e9b872c49c75852646f is 50, key is test_row_0/B:col10/1732416260986/Put/seqid=0 2024-11-24T02:44:22,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742302_1478 (size=12151) 2024-11-24T02:44:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:23,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b2931b404d0e4e9b872c49c75852646f 2024-11-24T02:44:23,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7593767812704631b2fa9273af8cd85d is 50, key is test_row_0/C:col10/1732416260986/Put/seqid=0 2024-11-24T02:44:23,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742303_1479 (size=12151) 2024-11-24T02:44:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:23,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:23,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416323187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416323188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416323188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416323189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416323190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416323299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416323309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416323309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416323309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416323309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416323508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416323519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416323519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416323519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416323519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7593767812704631b2fa9273af8cd85d 2024-11-24T02:44:23,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/f2028bf6416b4bdea820b01c44a8bd5a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a 2024-11-24T02:44:23,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a, entries=150, sequenceid=220, filesize=11.9 K 2024-11-24T02:44:23,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/b2931b404d0e4e9b872c49c75852646f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f 2024-11-24T02:44:23,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f, entries=150, sequenceid=220, filesize=11.9 K 2024-11-24T02:44:23,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7593767812704631b2fa9273af8cd85d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d 2024-11-24T02:44:23,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d, entries=150, sequenceid=220, filesize=11.9 K 2024-11-24T02:44:23,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cfb4c06d27130e960c1fcd48fcbed8af in 1252ms, sequenceid=220, compaction requested=false 2024-11-24T02:44:23,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:23,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:23,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T02:44:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T02:44:23,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T02:44:23,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8640 sec 2024-11-24T02:44:23,558 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8670 sec 2024-11-24T02:44:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:23,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:44:23,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:23,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:23,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:23,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:23,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:23,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:23,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/5e8cd39ab3d14e5c99bde3ce6399515a is 50, key is test_row_0/A:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742304_1480 (size=14541) 2024-11-24T02:44:23,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/5e8cd39ab3d14e5c99bde3ce6399515a 2024-11-24T02:44:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:44:23,840 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:23,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/cdd3cdf0878d483a99d192f6fa5b723f is 50, key is test_row_0/B:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:23,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T02:44:23,843 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:23,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:23,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:23,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742305_1481 (size=12151) 2024-11-24T02:44:23,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416323842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416323844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416323845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416323846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416323847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416323959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416323959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416323968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416323969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416323969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:23,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:23,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:23,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:24,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416324168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416324168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416324189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416324189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416324189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/cdd3cdf0878d483a99d192f6fa5b723f 2024-11-24T02:44:24,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7434d2ee643e42c4852fae216f85bbfe is 50, key is test_row_0/C:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:24,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742306_1482 (size=12151) 2024-11-24T02:44:24,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:24,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:24,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416324479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416324489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416324498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416324498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416324499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:24,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:24,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:24,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7434d2ee643e42c4852fae216f85bbfe 2024-11-24T02:44:24,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/5e8cd39ab3d14e5c99bde3ce6399515a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a 2024-11-24T02:44:24,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a, entries=200, sequenceid=240, filesize=14.2 K 2024-11-24T02:44:24,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/cdd3cdf0878d483a99d192f6fa5b723f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f 2024-11-24T02:44:24,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f, entries=150, sequenceid=240, filesize=11.9 K 2024-11-24T02:44:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/7434d2ee643e42c4852fae216f85bbfe as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe 2024-11-24T02:44:24,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe, entries=150, sequenceid=240, filesize=11.9 K 2024-11-24T02:44:24,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cfb4c06d27130e960c1fcd48fcbed8af in 848ms, sequenceid=240, compaction requested=true 2024-11-24T02:44:24,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:24,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:24,673 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:24,673 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,673 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,673 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/0a1e707d8f034920a0b7fb38c4dc0499, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=38.4 K 2024-11-24T02:44:24,673 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3336ec8e3b184ead8f93cf35e623bacf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.1 K 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a1e707d8f034920a0b7fb38c4dc0499, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3336ec8e3b184ead8f93cf35e623bacf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2028bf6416b4bdea820b01c44a8bd5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732416260958 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b2931b404d0e4e9b872c49c75852646f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732416260958 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8cd39ab3d14e5c99bde3ce6399515a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:24,674 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd3cdf0878d483a99d192f6fa5b723f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:24,679 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:24,679 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/44b6342e44df47cc8577d6771318baa7 is 50, key is test_row_0/A:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:24,683 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:24,683 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8d7bec2741494f82a3f3ac02e996e5af is 50, key is test_row_0/B:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:24,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742308_1484 (size=12765) 2024-11-24T02:44:24,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742307_1483 (size=12765) 2024-11-24T02:44:24,699 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/44b6342e44df47cc8577d6771318baa7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/44b6342e44df47cc8577d6771318baa7 2024-11-24T02:44:24,702 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 44b6342e44df47cc8577d6771318baa7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:24,702 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:24,702 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416264673; duration=0sec 2024-11-24T02:44:24,702 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:24,702 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:24,702 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:24,703 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:24,703 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:24,703 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,703 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/686c832f519d4e2aaf9319b4f8e08b03, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.1 K 2024-11-24T02:44:24,703 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686c832f519d4e2aaf9319b4f8e08b03, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732416260764 2024-11-24T02:44:24,703 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7593767812704631b2fa9273af8cd85d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732416260958 2024-11-24T02:44:24,703 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7434d2ee643e42c4852fae216f85bbfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:24,709 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:24,709 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/a6c8d350c65e43cc8dcf45f746995dc4 is 50, key is test_row_0/C:col10/1732416263189/Put/seqid=0 2024-11-24T02:44:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742309_1485 (size=12765) 2024-11-24T02:44:24,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:44:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:24,759 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:44:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:24,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:24,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:24,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:24,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:24,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/ce1377bb3b3e4594a2397bc10b108213 is 50, key is test_row_0/A:col10/1732416263845/Put/seqid=0 2024-11-24T02:44:24,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742310_1486 (size=12151) 2024-11-24T02:44:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:24,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:25,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416325029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416325030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416325031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416325032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416325033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,097 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8d7bec2741494f82a3f3ac02e996e5af as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8d7bec2741494f82a3f3ac02e996e5af 2024-11-24T02:44:25,102 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 8d7bec2741494f82a3f3ac02e996e5af(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:25,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:25,102 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416264673; duration=0sec 2024-11-24T02:44:25,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:25,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:25,119 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/a6c8d350c65e43cc8dcf45f746995dc4 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/a6c8d350c65e43cc8dcf45f746995dc4 2024-11-24T02:44:25,122 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into a6c8d350c65e43cc8dcf45f746995dc4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:25,122 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:25,122 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416264673; duration=0sec 2024-11-24T02:44:25,122 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:25,122 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:25,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416325139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416325149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416325149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416325149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416325149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/ce1377bb3b3e4594a2397bc10b108213 2024-11-24T02:44:25,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bb8a8de84454ec19e9abc0f73817634 is 50, key is test_row_0/B:col10/1732416263845/Put/seqid=0 2024-11-24T02:44:25,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742311_1487 (size=12151) 2024-11-24T02:44:25,176 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bb8a8de84454ec19e9abc0f73817634 2024-11-24T02:44:25,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/76a53affc0fc420f873e5b1dd8789f69 is 50, key is test_row_0/C:col10/1732416263845/Put/seqid=0 2024-11-24T02:44:25,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742312_1488 (size=12151) 2024-11-24T02:44:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416325349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416325359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416325359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416325368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416325368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,586 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/76a53affc0fc420f873e5b1dd8789f69 2024-11-24T02:44:25,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/ce1377bb3b3e4594a2397bc10b108213 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213 2024-11-24T02:44:25,592 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T02:44:25,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bb8a8de84454ec19e9abc0f73817634 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634 2024-11-24T02:44:25,596 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T02:44:25,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/76a53affc0fc420f873e5b1dd8789f69 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69 2024-11-24T02:44:25,599 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T02:44:25,600 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cfb4c06d27130e960c1fcd48fcbed8af in 841ms, sequenceid=254, compaction requested=false 2024-11-24T02:44:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T02:44:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T02:44:25,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T02:44:25,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7570 sec 2024-11-24T02:44:25,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7620 sec 2024-11-24T02:44:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:25,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:25,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:25,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/4ef44f2676444a51814590124d1e9dcf is 50, key is test_row_0/A:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:25,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742313_1489 (size=14741) 2024-11-24T02:44:25,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/4ef44f2676444a51814590124d1e9dcf 2024-11-24T02:44:25,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bec683fd27c4e35893c4cf5d044dd70 is 50, key is test_row_0/B:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:25,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742314_1490 (size=12301) 2024-11-24T02:44:25,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bec683fd27c4e35893c4cf5d044dd70 2024-11-24T02:44:25,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/ae2700d3518a4b1eb3878903a56d4bf8 is 50, key is test_row_0/C:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742315_1491 (size=12301) 2024-11-24T02:44:25,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416325689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416325690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416325690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416325691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416325691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416325809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416325809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416325809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416325809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:44:25,989 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:25,989 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T02:44:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:44:25,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:25,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:25,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:26,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416326019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416326019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416326019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416326029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/ae2700d3518a4b1eb3878903a56d4bf8 2024-11-24T02:44:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:44:26,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/4ef44f2676444a51814590124d1e9dcf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf 2024-11-24T02:44:26,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf, entries=200, sequenceid=276, filesize=14.4 K 2024-11-24T02:44:26,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/6bec683fd27c4e35893c4cf5d044dd70 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70 2024-11-24T02:44:26,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70, entries=150, sequenceid=276, filesize=12.0 K 2024-11-24T02:44:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/ae2700d3518a4b1eb3878903a56d4bf8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8 2024-11-24T02:44:26,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8, entries=150, sequenceid=276, filesize=12.0 K 2024-11-24T02:44:26,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cfb4c06d27130e960c1fcd48fcbed8af in 447ms, sequenceid=276, compaction requested=true 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:26,110 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:26,110 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:26,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:26,111 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:26,111 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:26,111 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:26,111 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:26,111 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:26,111 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:26,111 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8d7bec2741494f82a3f3ac02e996e5af, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.3 K 2024-11-24T02:44:26,111 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/44b6342e44df47cc8577d6771318baa7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=38.7 K 2024-11-24T02:44:26,111 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d7bec2741494f82a3f3ac02e996e5af, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:26,112 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b6342e44df47cc8577d6771318baa7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:26,112 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb8a8de84454ec19e9abc0f73817634, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416263837 2024-11-24T02:44:26,112 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce1377bb3b3e4594a2397bc10b108213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416263837 2024-11-24T02:44:26,112 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bec683fd27c4e35893c4cf5d044dd70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:26,112 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ef44f2676444a51814590124d1e9dcf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:26,118 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:26,118 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:26,118 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/ac04c553b52a4fb0b7c002ac9bdc76d3 is 50, key is test_row_0/A:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:26,118 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/49b5f8530e594044946bd08f13461192 is 50, key is test_row_0/B:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742316_1492 (size=13017) 2024-11-24T02:44:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742317_1493 (size=13017) 2024-11-24T02:44:26,124 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/ac04c553b52a4fb0b7c002ac9bdc76d3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ac04c553b52a4fb0b7c002ac9bdc76d3 2024-11-24T02:44:26,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T02:44:26,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:26,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:44:26,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:26,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:26,184 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into ac04c553b52a4fb0b7c002ac9bdc76d3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:26,184 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:26,184 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416266110; duration=0sec 2024-11-24T02:44:26,184 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:26,184 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:26,184 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:26,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:26,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:26,185 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:26,185 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/a6c8d350c65e43cc8dcf45f746995dc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.3 K 2024-11-24T02:44:26,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c8d350c65e43cc8dcf45f746995dc4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732416263174 2024-11-24T02:44:26,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/908f9e379ca94b2a856e7c854f7a981d is 50, key is test_row_0/A:col10/1732416265675/Put/seqid=0 2024-11-24T02:44:26,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a53affc0fc420f873e5b1dd8789f69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732416263837 2024-11-24T02:44:26,185 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae2700d3518a4b1eb3878903a56d4bf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:26,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742318_1494 (size=12301) 2024-11-24T02:44:26,188 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/908f9e379ca94b2a856e7c854f7a981d 2024-11-24T02:44:26,192 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:26,192 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/c49f5783dec540d6aa6dfdeb4d9490b0 is 50, key is test_row_0/C:col10/1732416265029/Put/seqid=0 2024-11-24T02:44:26,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/15d6759dd11e4fd694867b25c385997e is 50, key is test_row_0/B:col10/1732416265675/Put/seqid=0 2024-11-24T02:44:26,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742319_1495 (size=13017) 2024-11-24T02:44:26,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742320_1496 (size=12301) 2024-11-24T02:44:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:26,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:26,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416326305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:44:26,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416326329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416326329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416326329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416326349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416326409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,525 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/49b5f8530e594044946bd08f13461192 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/49b5f8530e594044946bd08f13461192 2024-11-24T02:44:26,528 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 49b5f8530e594044946bd08f13461192(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:26,528 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:26,528 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416266110; duration=0sec 2024-11-24T02:44:26,528 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:26,528 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/15d6759dd11e4fd694867b25c385997e 2024-11-24T02:44:26,604 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/c49f5783dec540d6aa6dfdeb4d9490b0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c49f5783dec540d6aa6dfdeb4d9490b0 2024-11-24T02:44:26,608 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into c49f5783dec540d6aa6dfdeb4d9490b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:26,608 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:26,608 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416266110; duration=0sec 2024-11-24T02:44:26,608 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:26,608 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f538f86d882241f89e92960f9bedc866 is 50, key is test_row_0/C:col10/1732416265675/Put/seqid=0 2024-11-24T02:44:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742321_1497 (size=12301) 2024-11-24T02:44:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:44:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416326628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416326848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416326848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416326848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416326869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416326938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:26,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,014 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f538f86d882241f89e92960f9bedc866 2024-11-24T02:44:27,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/908f9e379ca94b2a856e7c854f7a981d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d 2024-11-24T02:44:27,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:44:27,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/15d6759dd11e4fd694867b25c385997e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e 2024-11-24T02:44:27,024 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:44:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/f538f86d882241f89e92960f9bedc866 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866 2024-11-24T02:44:27,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T02:44:27,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cfb4c06d27130e960c1fcd48fcbed8af in 885ms, sequenceid=288, compaction requested=false 2024-11-24T02:44:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:27,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T02:44:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T02:44:27,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T02:44:27,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0390 sec 2024-11-24T02:44:27,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0430 sec 2024-11-24T02:44:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:44:27,129 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:27,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T02:44:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:44:27,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:27,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:27,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:44:27,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T02:44:27,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:27,283 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:27,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3608d1ac2532470c83cba92ac5e831b2 is 50, key is test_row_0/A:col10/1732416266304/Put/seqid=0 2024-11-24T02:44:27,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742322_1498 (size=12301) 2024-11-24T02:44:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:44:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:27,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416327516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416327629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,691 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3608d1ac2532470c83cba92ac5e831b2 2024-11-24T02:44:27,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eed7f2a500fa4cc6a3af59550d4a7f72 is 50, key is test_row_0/B:col10/1732416266304/Put/seqid=0 2024-11-24T02:44:27,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742323_1499 (size=12301) 2024-11-24T02:44:27,701 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eed7f2a500fa4cc6a3af59550d4a7f72 2024-11-24T02:44:27,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/9b72ec86591e4a6091500266e98248cb is 50, key is test_row_0/C:col10/1732416266304/Put/seqid=0 2024-11-24T02:44:27,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742324_1500 (size=12301) 2024-11-24T02:44:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:44:27,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416327839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416327859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,861 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416327869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,871 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,871 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416327889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:27,890 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_2' in cfb4c06d27130e960c1fcd48fcbed8af of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:28,114 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/9b72ec86591e4a6091500266e98248cb 2024-11-24T02:44:28,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/3608d1ac2532470c83cba92ac5e831b2 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2 2024-11-24T02:44:28,120 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T02:44:28,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/eed7f2a500fa4cc6a3af59550d4a7f72 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72 2024-11-24T02:44:28,123 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T02:44:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/9b72ec86591e4a6091500266e98248cb as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb 2024-11-24T02:44:28,126 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T02:44:28,127 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cfb4c06d27130e960c1fcd48fcbed8af in 844ms, sequenceid=311, compaction requested=true 2024-11-24T02:44:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T02:44:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T02:44:28,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T02:44:28,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 997 msec 2024-11-24T02:44:28,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0000 sec 2024-11-24T02:44:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:28,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:28,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:28,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2d69511b2fa84d98ab957dc7dd6d0b6c is 50, key is test_row_0/A:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:28,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742325_1501 (size=14741) 2024-11-24T02:44:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:44:28,269 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:28,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T02:44:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:28,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:28,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:28,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:28,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416328268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416328389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:28,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:44:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2d69511b2fa84d98ab957dc7dd6d0b6c 2024-11-24T02:44:28,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/5671b584e2864ffd941870b936074f53 is 50, key is test_row_0/B:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:28,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742326_1502 (size=12301) 2024-11-24T02:44:28,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:44:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:28,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:28,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416328609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:28,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:44:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:44:28,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:28,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:28,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416328919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:28,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/5671b584e2864ffd941870b936074f53 2024-11-24T02:44:28,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/8a3266e155c1466299784d2d4bfe0a83 is 50, key is test_row_0/C:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:28,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742327_1503 (size=12301) 2024-11-24T02:44:28,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/8a3266e155c1466299784d2d4bfe0a83 2024-11-24T02:44:28,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/2d69511b2fa84d98ab957dc7dd6d0b6c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c 2024-11-24T02:44:28,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c, entries=200, sequenceid=324, filesize=14.4 K 2024-11-24T02:44:28,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/5671b584e2864ffd941870b936074f53 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53 2024-11-24T02:44:29,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T02:44:29,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/8a3266e155c1466299784d2d4bfe0a83 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83 2024-11-24T02:44:29,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T02:44:29,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cfb4c06d27130e960c1fcd48fcbed8af in 856ms, sequenceid=324, compaction requested=true 2024-11-24T02:44:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:29,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:29,009 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:29,009 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:29,011 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:29,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:29,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:29,011 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:29,011 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/49b5f8530e594044946bd08f13461192, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=48.8 K 2024-11-24T02:44:29,011 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:29,011 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:29,011 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ac04c553b52a4fb0b7c002ac9bdc76d3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=51.1 K 2024-11-24T02:44:29,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac04c553b52a4fb0b7c002ac9bdc76d3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:29,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908f9e379ca94b2a856e7c854f7a981d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416265675 2024-11-24T02:44:29,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3608d1ac2532470c83cba92ac5e831b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732416266282 2024-11-24T02:44:29,012 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d69511b2fa84d98ab957dc7dd6d0b6c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267466 2024-11-24T02:44:29,013 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b5f8530e594044946bd08f13461192, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:29,013 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d6759dd11e4fd694867b25c385997e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416265675 2024-11-24T02:44:29,014 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting eed7f2a500fa4cc6a3af59550d4a7f72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732416266282 2024-11-24T02:44:29,014 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5671b584e2864ffd941870b936074f53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267495 2024-11-24T02:44:29,019 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:29,019 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:29,020 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/24352dbe45e84fc1b2aaa2a7356ca0a3 is 50, key is test_row_0/B:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:29,020 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/17370dc1b9e04bbb9d284cb9bf87a16f is 50, key is test_row_0/A:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:29,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742328_1504 (size=13153) 2024-11-24T02:44:29,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742329_1505 (size=13153) 2024-11-24T02:44:29,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:29,035 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:29,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/51de7b48693d484797e2a167d2d162ad is 50, key is test_row_0/A:col10/1732416268187/Put/seqid=0 2024-11-24T02:44:29,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742330_1506 (size=12301) 2024-11-24T02:44:29,042 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/51de7b48693d484797e2a167d2d162ad 2024-11-24T02:44:29,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8524f7d66e374725a8205e1b53ceb049 is 50, key is test_row_0/B:col10/1732416268187/Put/seqid=0 2024-11-24T02:44:29,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742331_1507 (size=12301) 2024-11-24T02:44:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:29,432 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/24352dbe45e84fc1b2aaa2a7356ca0a3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/24352dbe45e84fc1b2aaa2a7356ca0a3 2024-11-24T02:44:29,432 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/17370dc1b9e04bbb9d284cb9bf87a16f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/17370dc1b9e04bbb9d284cb9bf87a16f 2024-11-24T02:44:29,435 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into 17370dc1b9e04bbb9d284cb9bf87a16f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:29,435 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:29,435 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=12, startTime=1732416269008; duration=0sec 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:29,436 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 24352dbe45e84fc1b2aaa2a7356ca0a3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:29,436 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=12, startTime=1732416269008; duration=0sec 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:29,436 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:29,437 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:29,437 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:29,437 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c49f5783dec540d6aa6dfdeb4d9490b0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=48.8 K 2024-11-24T02:44:29,437 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49f5783dec540d6aa6dfdeb4d9490b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732416265028 2024-11-24T02:44:29,437 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f538f86d882241f89e92960f9bedc866, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732416265675 2024-11-24T02:44:29,437 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b72ec86591e4a6091500266e98248cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732416266282 2024-11-24T02:44:29,438 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3266e155c1466299784d2d4bfe0a83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267495 2024-11-24T02:44:29,444 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:29,445 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/c130b1b0fad5478cb0b40bb736890232 is 50, key is test_row_0/C:col10/1732416267514/Put/seqid=0 2024-11-24T02:44:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742332_1508 (size=13153) 2024-11-24T02:44:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:29,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:29,452 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8524f7d66e374725a8205e1b53ceb049 2024-11-24T02:44:29,452 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/c130b1b0fad5478cb0b40bb736890232 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c130b1b0fad5478cb0b40bb736890232 2024-11-24T02:44:29,456 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into c130b1b0fad5478cb0b40bb736890232(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:29,456 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:29,456 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=12, startTime=1732416269009; duration=0sec 2024-11-24T02:44:29,456 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:29,456 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:29,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/68a42c129a3448d6aa4c7d1052b95099 is 50, key is test_row_0/C:col10/1732416268187/Put/seqid=0 2024-11-24T02:44:29,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742333_1509 (size=12301) 2024-11-24T02:44:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416329509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416329619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416329829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/68a42c129a3448d6aa4c7d1052b95099 2024-11-24T02:44:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/51de7b48693d484797e2a167d2d162ad as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad 2024-11-24T02:44:29,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T02:44:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8524f7d66e374725a8205e1b53ceb049 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049 2024-11-24T02:44:29,871 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T02:44:29,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/68a42c129a3448d6aa4c7d1052b95099 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099 2024-11-24T02:44:29,874 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T02:44:29,875 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cfb4c06d27130e960c1fcd48fcbed8af in 840ms, sequenceid=343, compaction requested=false 2024-11-24T02:44:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:29,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T02:44:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T02:44:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:29,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:29,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T02:44:29,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-11-24T02:44:29,879 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.6090 sec 2024-11-24T02:44:29,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/fe3bbee7302a435d981745f28965936b is 50, key is test_row_0/A:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:29,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742334_1510 (size=14741) 2024-11-24T02:44:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416329910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416329911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416329912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416329916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416330029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416330029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416330029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416330029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416330139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416330239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416330239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416330239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416330239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/fe3bbee7302a435d981745f28965936b 2024-11-24T02:44:30,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/c85ae050d83f4328b991c04469286f37 is 50, key is test_row_0/B:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:30,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742335_1511 (size=12301) 2024-11-24T02:44:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:44:30,419 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:30,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T02:44:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:30,421 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:30,422 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:30,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416330549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416330549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416330558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416330558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:44:30,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44454 deadline: 1732416330649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:30,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/c85ae050d83f4328b991c04469286f37 2024-11-24T02:44:30,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/787ff9076a2c42edb3ede4ee9d70af9b is 50, key is test_row_0/C:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:30,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742336_1512 (size=12301) 2024-11-24T02:44:30,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:44:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:30,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:44:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:30,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:30,921 INFO [Thread-1944 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1944. 2024-11-24T02:44:30,921 DEBUG [Thread-1944 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:30,921 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,921 DEBUG [Thread-1944 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:30,926 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-24T02:44:30,926 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:30,926 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,926 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:30,927 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-24T02:44:30,927 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:30,927 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,927 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:30,933 INFO [Thread-1946 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1946. 2024-11-24T02:44:30,933 DEBUG [Thread-1946 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:30,933 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,933 DEBUG [Thread-1946 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:30,937 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-24T02:44:30,937 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:30,937 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,937 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:30,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:31,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:44:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:31,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44414 deadline: 1732416331060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44452 deadline: 1732416331060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:31,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44426 deadline: 1732416331069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:31,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44444 deadline: 1732416331079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:44:31,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/787ff9076a2c42edb3ede4ee9d70af9b 2024-11-24T02:44:31,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/fe3bbee7302a435d981745f28965936b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b 2024-11-24T02:44:31,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b, entries=200, sequenceid=360, filesize=14.4 K 2024-11-24T02:44:31,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/c85ae050d83f4328b991c04469286f37 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37 2024-11-24T02:44:31,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37, entries=150, sequenceid=360, filesize=12.0 K 2024-11-24T02:44:31,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/787ff9076a2c42edb3ede4ee9d70af9b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b 2024-11-24T02:44:31,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b, entries=150, sequenceid=360, filesize=12.0 K 2024-11-24T02:44:31,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for cfb4c06d27130e960c1fcd48fcbed8af in 1250ms, sequenceid=360, compaction requested=true 2024-11-24T02:44:31,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:31,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:31,126 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:31,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:31,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:31,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cfb4c06d27130e960c1fcd48fcbed8af:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:31,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/B is initiating minor compaction (all files) 2024-11-24T02:44:31,127 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/A is initiating minor compaction (all files) 2024-11-24T02:44:31,127 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/B in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,127 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/A in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,127 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/24352dbe45e84fc1b2aaa2a7356ca0a3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.9 K 2024-11-24T02:44:31,127 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/17370dc1b9e04bbb9d284cb9bf87a16f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=39.3 K 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 24352dbe45e84fc1b2aaa2a7356ca0a3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267495 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17370dc1b9e04bbb9d284cb9bf87a16f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267495 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8524f7d66e374725a8205e1b53ceb049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732416268187 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51de7b48693d484797e2a167d2d162ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732416268187 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c85ae050d83f4328b991c04469286f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416269496 2024-11-24T02:44:31,128 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3bbee7302a435d981745f28965936b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416269487 2024-11-24T02:44:31,183 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:31,183 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/b4059b0f17e84cc0b4e84076d23da1c0 is 50, key is test_row_0/A:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:31,184 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#B#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:31,184 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/2ec4432888fa48e0b19454e441e4f1c1 is 50, key is test_row_0/B:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:31,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,186 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:31,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:31,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742337_1513 (size=13255) 2024-11-24T02:44:31,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:31,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742338_1514 (size=13255) 2024-11-24T02:44:31,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9383c65dd9c3465c9c54e9f9b7439cba is 50, key is test_row_0/A:col10/1732416269907/Put/seqid=0 2024-11-24T02:44:31,192 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/2ec4432888fa48e0b19454e441e4f1c1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/2ec4432888fa48e0b19454e441e4f1c1 2024-11-24T02:44:31,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742339_1515 (size=12301) 2024-11-24T02:44:31,194 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9383c65dd9c3465c9c54e9f9b7439cba 2024-11-24T02:44:31,195 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/B of cfb4c06d27130e960c1fcd48fcbed8af into 2ec4432888fa48e0b19454e441e4f1c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:31,195 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:31,196 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/B, priority=13, startTime=1732416271127; duration=0sec 2024-11-24T02:44:31,196 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:31,196 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:B 2024-11-24T02:44:31,196 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:31,197 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:31,197 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): cfb4c06d27130e960c1fcd48fcbed8af/C is initiating minor compaction (all files) 2024-11-24T02:44:31,197 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cfb4c06d27130e960c1fcd48fcbed8af/C in TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:31,197 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c130b1b0fad5478cb0b40bb736890232, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp, totalSize=36.9 K 2024-11-24T02:44:31,197 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c130b1b0fad5478cb0b40bb736890232, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732416267495 2024-11-24T02:44:31,197 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a42c129a3448d6aa4c7d1052b95099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732416268187 2024-11-24T02:44:31,198 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 787ff9076a2c42edb3ede4ee9d70af9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1732416269496 2024-11-24T02:44:31,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/565aa5ee5d6c4ea2b02ee601af7909be is 50, key is test_row_0/B:col10/1732416269907/Put/seqid=0 2024-11-24T02:44:31,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742340_1516 (size=12301) 2024-11-24T02:44:31,203 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cfb4c06d27130e960c1fcd48fcbed8af#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:31,203 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/ac94133e2e324409a6f1d92116e1f475 is 50, key is test_row_0/C:col10/1732416269507/Put/seqid=0 2024-11-24T02:44:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742341_1517 (size=13255) 2024-11-24T02:44:31,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:44:31,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:31,597 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/b4059b0f17e84cc0b4e84076d23da1c0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b4059b0f17e84cc0b4e84076d23da1c0 2024-11-24T02:44:31,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/565aa5ee5d6c4ea2b02ee601af7909be 2024-11-24T02:44:31,604 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/A of cfb4c06d27130e960c1fcd48fcbed8af into b4059b0f17e84cc0b4e84076d23da1c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:31,604 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:31,604 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/A, priority=13, startTime=1732416271126; duration=0sec 2024-11-24T02:44:31,604 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:31,604 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:A 2024-11-24T02:44:31,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/77059328896941b3b6e41f9473e2066e is 50, key is test_row_0/C:col10/1732416269907/Put/seqid=0 2024-11-24T02:44:31,609 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/ac94133e2e324409a6f1d92116e1f475 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ac94133e2e324409a6f1d92116e1f475 2024-11-24T02:44:31,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742342_1518 (size=12301) 2024-11-24T02:44:31,612 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cfb4c06d27130e960c1fcd48fcbed8af/C of cfb4c06d27130e960c1fcd48fcbed8af into ac94133e2e324409a6f1d92116e1f475(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:31,612 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:31,612 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., storeName=cfb4c06d27130e960c1fcd48fcbed8af/C, priority=13, startTime=1732416271127; duration=0sec 2024-11-24T02:44:31,612 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:31,612 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cfb4c06d27130e960c1fcd48fcbed8af:C 2024-11-24T02:44:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:31,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. as already flushing 2024-11-24T02:44:31,670 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-24T02:44:31,670 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:31,670 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:31,670 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:31,670 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,011 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/77059328896941b3b6e41f9473e2066e 2024-11-24T02:44:32,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9383c65dd9c3465c9c54e9f9b7439cba as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9383c65dd9c3465c9c54e9f9b7439cba 2024-11-24T02:44:32,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9383c65dd9c3465c9c54e9f9b7439cba, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T02:44:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/565aa5ee5d6c4ea2b02ee601af7909be as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/565aa5ee5d6c4ea2b02ee601af7909be 2024-11-24T02:44:32,028 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/565aa5ee5d6c4ea2b02ee601af7909be, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T02:44:32,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/77059328896941b3b6e41f9473e2066e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/77059328896941b3b6e41f9473e2066e 2024-11-24T02:44:32,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/77059328896941b3b6e41f9473e2066e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T02:44:32,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=6.71 KB/6870 for cfb4c06d27130e960c1fcd48fcbed8af in 848ms, sequenceid=378, compaction requested=false 2024-11-24T02:44:32,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for cfb4c06d27130e960c1fcd48fcbed8af: 2024-11-24T02:44:32,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:32,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T02:44:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T02:44:32,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T02:44:32,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-24T02:44:32,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6190 sec 2024-11-24T02:44:32,071 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-11-24T02:44:32,072 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:32,072 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,072 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,083 INFO [Thread-1935 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1935. 2024-11-24T02:44:32,083 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-11-24T02:44:32,083 DEBUG [Thread-1935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:32,083 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,083 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:32,083 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,083 DEBUG [Thread-1935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,084 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,101 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-11-24T02:44:32,101 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:32,101 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,101 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,102 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:44:32,570 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1328 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3983 rows 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1325 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3974 rows 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1325 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3974 rows 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1328 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3982 rows 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1330 2024-11-24T02:44:32,570 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3986 rows 2024-11-24T02:44:32,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:44:32,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:44:32,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:32,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:32,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:44:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:32,574 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416272574"}]},"ts":"1732416272574"} 2024-11-24T02:44:32,575 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:44:32,575 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:44:32,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:44:32,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, UNASSIGN}] 2024-11-24T02:44:32,577 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, UNASSIGN 2024-11-24T02:44:32,578 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=cfb4c06d27130e960c1fcd48fcbed8af, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:32,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, UNASSIGN because future has completed 2024-11-24T02:44:32,579 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:44:32,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:32,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:32,732 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:32,732 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:44:32,732 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing cfb4c06d27130e960c1fcd48fcbed8af, disabling compactions & flushes 2024-11-24T02:44:32,732 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:32,733 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:32,733 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. after waiting 0 ms 2024-11-24T02:44:32,733 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:32,733 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing cfb4c06d27130e960c1fcd48fcbed8af 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=A 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=B 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cfb4c06d27130e960c1fcd48fcbed8af, store=C 2024-11-24T02:44:32,734 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:32,740 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9165c866b1ed431bb7547704e306884f is 50, key is test_row_0/A:col10/1732416272080/Put/seqid=0 2024-11-24T02:44:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742343_1519 (size=12301) 2024-11-24T02:44:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:33,146 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9165c866b1ed431bb7547704e306884f 2024-11-24T02:44:33,160 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8adad6e99e214f6db6c38a6847634a97 is 50, key is test_row_0/B:col10/1732416272080/Put/seqid=0 2024-11-24T02:44:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742344_1520 (size=12301) 2024-11-24T02:44:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:33,566 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8adad6e99e214f6db6c38a6847634a97 2024-11-24T02:44:33,579 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/584706d07b044c498008ae81e441b5c6 is 50, key is test_row_0/C:col10/1732416272080/Put/seqid=0 2024-11-24T02:44:33,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742345_1521 (size=12301) 2024-11-24T02:44:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:33,983 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/584706d07b044c498008ae81e441b5c6 2024-11-24T02:44:33,988 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/A/9165c866b1ed431bb7547704e306884f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9165c866b1ed431bb7547704e306884f 2024-11-24T02:44:33,992 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9165c866b1ed431bb7547704e306884f, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T02:44:33,994 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/B/8adad6e99e214f6db6c38a6847634a97 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8adad6e99e214f6db6c38a6847634a97 2024-11-24T02:44:33,999 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8adad6e99e214f6db6c38a6847634a97, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T02:44:34,000 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/.tmp/C/584706d07b044c498008ae81e441b5c6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/584706d07b044c498008ae81e441b5c6 2024-11-24T02:44:34,005 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/584706d07b044c498008ae81e441b5c6, entries=150, sequenceid=389, filesize=12.0 K 2024-11-24T02:44:34,005 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cfb4c06d27130e960c1fcd48fcbed8af in 1272ms, sequenceid=389, compaction requested=true 2024-11-24T02:44:34,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/20faa751e15d404a9c5e0ee9c83d1602, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/97532476f7834de79a03ca3e80076ae7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/55e023bcc6404bc4a2a1c4b7792c987e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/1f97f29626564aa592f14134c5bda5e4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/0a1e707d8f034920a0b7fb38c4dc0499, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/44b6342e44df47cc8577d6771318baa7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ac04c553b52a4fb0b7c002ac9bdc76d3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/17370dc1b9e04bbb9d284cb9bf87a16f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b] to archive 2024-11-24T02:44:34,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:34,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/d073ce92962744b2b1cd00aaae072244 2024-11-24T02:44:34,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2580441f1a934fe582e4b6c345366e9c 2024-11-24T02:44:34,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/c94f75af511641eca8767f67daa8067e 2024-11-24T02:44:34,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/20faa751e15d404a9c5e0ee9c83d1602 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/20faa751e15d404a9c5e0ee9c83d1602 2024-11-24T02:44:34,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/790a927c33da4bf59611c90f640dcc33 2024-11-24T02:44:34,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/39f47cc2fa60428c9ea0781d8da895c7 2024-11-24T02:44:34,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/97532476f7834de79a03ca3e80076ae7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/97532476f7834de79a03ca3e80076ae7 2024-11-24T02:44:34,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3727dd87c0ff4ebe949463c8badc44f7 2024-11-24T02:44:34,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/13f72177977e45e784439a3a913de592 2024-11-24T02:44:34,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/55e023bcc6404bc4a2a1c4b7792c987e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/55e023bcc6404bc4a2a1c4b7792c987e 2024-11-24T02:44:34,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/bddbe55bd78c483da2b30919c3a5fcc4 2024-11-24T02:44:34,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2cd24d46b9924451896aaeef5d4f1698 2024-11-24T02:44:34,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/1f97f29626564aa592f14134c5bda5e4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/1f97f29626564aa592f14134c5bda5e4 2024-11-24T02:44:34,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/82065058804a4b6bbbf7a72ad36fcb43 2024-11-24T02:44:34,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b3ccf15146c44532a60a59da9720eda7 2024-11-24T02:44:34,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/0a1e707d8f034920a0b7fb38c4dc0499 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/0a1e707d8f034920a0b7fb38c4dc0499 2024-11-24T02:44:34,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/f2028bf6416b4bdea820b01c44a8bd5a 2024-11-24T02:44:34,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/5e8cd39ab3d14e5c99bde3ce6399515a 2024-11-24T02:44:34,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/44b6342e44df47cc8577d6771318baa7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/44b6342e44df47cc8577d6771318baa7 2024-11-24T02:44:34,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ce1377bb3b3e4594a2397bc10b108213 2024-11-24T02:44:34,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/4ef44f2676444a51814590124d1e9dcf 2024-11-24T02:44:34,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ac04c553b52a4fb0b7c002ac9bdc76d3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/ac04c553b52a4fb0b7c002ac9bdc76d3 2024-11-24T02:44:34,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/908f9e379ca94b2a856e7c854f7a981d 2024-11-24T02:44:34,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/3608d1ac2532470c83cba92ac5e831b2 2024-11-24T02:44:34,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/2d69511b2fa84d98ab957dc7dd6d0b6c 2024-11-24T02:44:34,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/17370dc1b9e04bbb9d284cb9bf87a16f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/17370dc1b9e04bbb9d284cb9bf87a16f 2024-11-24T02:44:34,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/51de7b48693d484797e2a167d2d162ad 2024-11-24T02:44:34,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/fe3bbee7302a435d981745f28965936b 2024-11-24T02:44:34,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/32468a1274944049ac6f5853be88a8b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3a86d90678a84973958efa5d1bfef47b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fac5316f55d0474cadfe917d0aa56932, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5aa06c93fcfc42c5a6006e4a7295d2ca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3336ec8e3b184ead8f93cf35e623bacf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8d7bec2741494f82a3f3ac02e996e5af, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/49b5f8530e594044946bd08f13461192, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/24352dbe45e84fc1b2aaa2a7356ca0a3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37] to archive 2024-11-24T02:44:34,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:34,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6a5439d6efb24597830d29d31a26f94f 2024-11-24T02:44:34,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fb8f24758a5a405db186ab0cdd4c7f95 2024-11-24T02:44:34,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/32468a1274944049ac6f5853be88a8b9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/32468a1274944049ac6f5853be88a8b9 2024-11-24T02:44:34,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/117f7280b82e4654aabc4922f380bdc2 2024-11-24T02:44:34,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/015a143c5d394532b854ac06fa63a7a4 2024-11-24T02:44:34,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3a86d90678a84973958efa5d1bfef47b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3a86d90678a84973958efa5d1bfef47b 2024-11-24T02:44:34,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/da5582eebc934a23b7a05a6223fef83a 2024-11-24T02:44:34,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b55ee07207014672a71941028f8f0c33 2024-11-24T02:44:34,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fac5316f55d0474cadfe917d0aa56932 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/fac5316f55d0474cadfe917d0aa56932 2024-11-24T02:44:34,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/231a618dad2d4e2f9d2c19790a5c8b0a 2024-11-24T02:44:34,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/1e7e757f84474c8189b24d831fb02a5a 2024-11-24T02:44:34,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5aa06c93fcfc42c5a6006e4a7295d2ca to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5aa06c93fcfc42c5a6006e4a7295d2ca 2024-11-24T02:44:34,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eec9816f50b74efba53b2ff5a184ec80 2024-11-24T02:44:34,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/e13ac1f082e54100a388d0926370341b 2024-11-24T02:44:34,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3336ec8e3b184ead8f93cf35e623bacf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/3336ec8e3b184ead8f93cf35e623bacf 2024-11-24T02:44:34,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/70d6e43f708a45d49ad5c16f1c3b944b 2024-11-24T02:44:34,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/b2931b404d0e4e9b872c49c75852646f 2024-11-24T02:44:34,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8d7bec2741494f82a3f3ac02e996e5af to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8d7bec2741494f82a3f3ac02e996e5af 2024-11-24T02:44:34,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/cdd3cdf0878d483a99d192f6fa5b723f 2024-11-24T02:44:34,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bb8a8de84454ec19e9abc0f73817634 2024-11-24T02:44:34,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/49b5f8530e594044946bd08f13461192 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/49b5f8530e594044946bd08f13461192 2024-11-24T02:44:34,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/6bec683fd27c4e35893c4cf5d044dd70 2024-11-24T02:44:34,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/15d6759dd11e4fd694867b25c385997e 2024-11-24T02:44:34,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/eed7f2a500fa4cc6a3af59550d4a7f72 2024-11-24T02:44:34,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/24352dbe45e84fc1b2aaa2a7356ca0a3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/24352dbe45e84fc1b2aaa2a7356ca0a3 2024-11-24T02:44:34,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/5671b584e2864ffd941870b936074f53 2024-11-24T02:44:34,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8524f7d66e374725a8205e1b53ceb049 2024-11-24T02:44:34,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/c85ae050d83f4328b991c04469286f37 2024-11-24T02:44:34,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cc982fac096b4ce5ad3f4366f2fc537b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4f4bcca38a2b4309a0cc3e76f5d302ee, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/1766617e9eac4a88a6ecf8fd3e90d40f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/b11e98599e0446bba127bd332b6d0a34, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/686c832f519d4e2aaf9319b4f8e08b03, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/a6c8d350c65e43cc8dcf45f746995dc4, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c49f5783dec540d6aa6dfdeb4d9490b0, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c130b1b0fad5478cb0b40bb736890232, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b] to archive 2024-11-24T02:44:34,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:44:34,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/48ac6e47fa934dbab92d26440823034d 2024-11-24T02:44:34,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/3f3669deff6f4e2bbf7facadddd6a79f 2024-11-24T02:44:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cc982fac096b4ce5ad3f4366f2fc537b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cc982fac096b4ce5ad3f4366f2fc537b 2024-11-24T02:44:34,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/cca457bfc7f146a3a5fc7661ba7e20f1 2024-11-24T02:44:34,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/96a6d2c18b904ae3a576433504ad0270 2024-11-24T02:44:34,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4f4bcca38a2b4309a0cc3e76f5d302ee to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4f4bcca38a2b4309a0cc3e76f5d302ee 2024-11-24T02:44:34,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/5255e53bb567473bb5ad2fb6ce14b9c7 2024-11-24T02:44:34,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/201621d1d13341fa9eee481a8ec2fd57 2024-11-24T02:44:34,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/1766617e9eac4a88a6ecf8fd3e90d40f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/1766617e9eac4a88a6ecf8fd3e90d40f 2024-11-24T02:44:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/965eefb7bf364c92a04230a173fb5e7d 2024-11-24T02:44:34,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7b2b91a1efb7455db9a0b5d6d3ddd02d 2024-11-24T02:44:34,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/b11e98599e0446bba127bd332b6d0a34 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/b11e98599e0446bba127bd332b6d0a34 2024-11-24T02:44:34,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/6e1fa747fab6418babe38ce5719b86a4 2024-11-24T02:44:34,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/4cd488e22ed4408c936eb8e1af02ab29 2024-11-24T02:44:34,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/686c832f519d4e2aaf9319b4f8e08b03 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/686c832f519d4e2aaf9319b4f8e08b03 2024-11-24T02:44:34,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f51faa82851b422cafef5c4a108c1ba0 2024-11-24T02:44:34,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7593767812704631b2fa9273af8cd85d 2024-11-24T02:44:34,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/a6c8d350c65e43cc8dcf45f746995dc4 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/a6c8d350c65e43cc8dcf45f746995dc4 2024-11-24T02:44:34,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/7434d2ee643e42c4852fae216f85bbfe 2024-11-24T02:44:34,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/76a53affc0fc420f873e5b1dd8789f69 2024-11-24T02:44:34,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c49f5783dec540d6aa6dfdeb4d9490b0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c49f5783dec540d6aa6dfdeb4d9490b0 2024-11-24T02:44:34,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ae2700d3518a4b1eb3878903a56d4bf8 2024-11-24T02:44:34,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/f538f86d882241f89e92960f9bedc866 2024-11-24T02:44:34,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/9b72ec86591e4a6091500266e98248cb 2024-11-24T02:44:34,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c130b1b0fad5478cb0b40bb736890232 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/c130b1b0fad5478cb0b40bb736890232 2024-11-24T02:44:34,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/8a3266e155c1466299784d2d4bfe0a83 2024-11-24T02:44:34,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/68a42c129a3448d6aa4c7d1052b95099 2024-11-24T02:44:34,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/787ff9076a2c42edb3ede4ee9d70af9b 2024-11-24T02:44:34,076 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/recovered.edits/392.seqid, newMaxSeqId=392, maxSeqId=1 2024-11-24T02:44:34,077 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. 2024-11-24T02:44:34,077 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for cfb4c06d27130e960c1fcd48fcbed8af: Waiting for close lock at 1732416272732Running coprocessor pre-close hooks at 1732416272732Disabling compacts and flushes for region at 1732416272732Disabling writes for close at 1732416272733 (+1 ms)Obtaining lock to block concurrent updates at 1732416272733Preparing flush snapshotting stores in cfb4c06d27130e960c1fcd48fcbed8af at 1732416272733Finished memstore snapshotting TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732416272734 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af. at 1732416272736 (+2 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/A: creating writer at 1732416272736Flushing cfb4c06d27130e960c1fcd48fcbed8af/A: appending metadata at 1732416272740 (+4 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/A: closing flushed file at 1732416272740Flushing cfb4c06d27130e960c1fcd48fcbed8af/B: creating writer at 1732416273155 (+415 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/B: appending metadata at 1732416273160 (+5 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/B: closing flushed file at 1732416273160Flushing cfb4c06d27130e960c1fcd48fcbed8af/C: creating writer at 1732416273576 (+416 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/C: appending metadata at 1732416273579 (+3 ms)Flushing cfb4c06d27130e960c1fcd48fcbed8af/C: closing flushed file at 1732416273579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4470006b: reopening flushed file at 1732416273988 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77cee53a: reopening flushed file at 1732416273992 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@527a0e5b: reopening flushed file at 1732416273999 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for cfb4c06d27130e960c1fcd48fcbed8af in 1272ms, sequenceid=389, compaction requested=true at 1732416274005 (+6 ms)Writing region close event to WAL at 1732416274074 (+69 ms)Running coprocessor post-close hooks at 1732416274077 (+3 ms)Closed at 1732416274077 2024-11-24T02:44:34,078 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:34,078 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=cfb4c06d27130e960c1fcd48fcbed8af, regionState=CLOSED 2024-11-24T02:44:34,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:34,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T02:44:34,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure cfb4c06d27130e960c1fcd48fcbed8af, server=7c69a60bd8f6,34251,1732416122955 in 1.5010 sec 2024-11-24T02:44:34,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-24T02:44:34,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cfb4c06d27130e960c1fcd48fcbed8af, UNASSIGN in 1.5040 sec 2024-11-24T02:44:34,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T02:44:34,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-24T02:44:34,085 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416274085"}]},"ts":"1732416274085"} 2024-11-24T02:44:34,086 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:44:34,086 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:44:34,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-24T02:44:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:44:34,719 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:34,720 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:44:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,723 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:44:34,724 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,728 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:34,732 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/recovered.edits] 2024-11-24T02:44:34,736 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9165c866b1ed431bb7547704e306884f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9165c866b1ed431bb7547704e306884f 2024-11-24T02:44:34,738 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9383c65dd9c3465c9c54e9f9b7439cba to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/9383c65dd9c3465c9c54e9f9b7439cba 2024-11-24T02:44:34,740 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b4059b0f17e84cc0b4e84076d23da1c0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/A/b4059b0f17e84cc0b4e84076d23da1c0 2024-11-24T02:44:34,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/2ec4432888fa48e0b19454e441e4f1c1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/2ec4432888fa48e0b19454e441e4f1c1 2024-11-24T02:44:34,747 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/565aa5ee5d6c4ea2b02ee601af7909be to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/565aa5ee5d6c4ea2b02ee601af7909be 2024-11-24T02:44:34,749 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8adad6e99e214f6db6c38a6847634a97 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/B/8adad6e99e214f6db6c38a6847634a97 2024-11-24T02:44:34,752 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/584706d07b044c498008ae81e441b5c6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/584706d07b044c498008ae81e441b5c6 2024-11-24T02:44:34,754 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/77059328896941b3b6e41f9473e2066e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/77059328896941b3b6e41f9473e2066e 2024-11-24T02:44:34,756 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ac94133e2e324409a6f1d92116e1f475 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/C/ac94133e2e324409a6f1d92116e1f475 2024-11-24T02:44:34,759 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/recovered.edits/392.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af/recovered.edits/392.seqid 2024-11-24T02:44:34,760 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/cfb4c06d27130e960c1fcd48fcbed8af 2024-11-24T02:44:34,760 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:44:34,764 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,767 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:44:34,770 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:44:34,771 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,772 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:44:34,772 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416274772"}]},"ts":"9223372036854775807"} 2024-11-24T02:44:34,774 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:44:34,774 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cfb4c06d27130e960c1fcd48fcbed8af, NAME => 'TestAcidGuarantees,,1732416248714.cfb4c06d27130e960c1fcd48fcbed8af.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:44:34,774 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:44:34,774 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416274774"}]},"ts":"9223372036854775807"} 2024-11-24T02:44:34,776 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:44:34,777 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-24T02:44:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:44:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:44:34,830 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:34,845 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=347 (was 333) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10061 (was 10074) 2024-11-24T02:44:34,854 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=347, ProcessCount=11, AvailableMemoryMB=10061 2024-11-24T02:44:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:44:34,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:44:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:34,857 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:44:34,857 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:34,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-24T02:44:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:34,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:44:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742346_1522 (size=1061) 2024-11-24T02:44:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:35,270 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6 2024-11-24T02:44:35,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742347_1523 (size=53) 2024-11-24T02:44:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:35,680 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:44:35,681 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fa18a3ad7ebd64590be10753e6ed8f58, disabling compactions & flushes 2024-11-24T02:44:35,681 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:35,681 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:35,681 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. after waiting 0 ms 2024-11-24T02:44:35,681 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:35,681 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:35,681 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa18a3ad7ebd64590be10753e6ed8f58: Waiting for close lock at 1732416275681Disabling compacts and flushes for region at 1732416275681Disabling writes for close at 1732416275681Writing region close event to WAL at 1732416275681Closed at 1732416275681 2024-11-24T02:44:35,684 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:44:35,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416275684"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416275684"}]},"ts":"1732416275684"} 2024-11-24T02:44:35,688 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:44:35,689 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:44:35,689 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416275689"}]},"ts":"1732416275689"} 2024-11-24T02:44:35,691 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:44:35,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, ASSIGN}] 2024-11-24T02:44:35,692 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, ASSIGN 2024-11-24T02:44:35,693 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=false 2024-11-24T02:44:35,843 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:35,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, ASSIGN because future has completed 2024-11-24T02:44:35,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:36,000 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:36,000 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:44:36,000 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,000 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:44:36,000 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,000 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,001 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,002 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:36,002 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName A 2024-11-24T02:44:36,002 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:36,002 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:36,003 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,003 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:36,003 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName B 2024-11-24T02:44:36,003 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:36,004 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:36,004 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,004 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:36,004 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName C 2024-11-24T02:44:36,004 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:36,005 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:36,005 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:36,005 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,005 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,005 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,006 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,006 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,007 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:44:36,007 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,009 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:44:36,009 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened fa18a3ad7ebd64590be10753e6ed8f58; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71105664, jitterRate=0.05955696105957031}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:44:36,009 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:36,009 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for fa18a3ad7ebd64590be10753e6ed8f58: Running coprocessor pre-open hook at 1732416276000Writing region info on filesystem at 1732416276000Initializing all the Stores at 1732416276001 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416276001Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416276001Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416276001Cleaning up temporary data from old regions at 1732416276006 (+5 ms)Running coprocessor post-open hooks at 1732416276009 (+3 ms)Region opened successfully at 1732416276009 2024-11-24T02:44:36,010 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., pid=144, masterSystemTime=1732416275997 2024-11-24T02:44:36,011 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:36,011 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:36,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:36,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:36,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-24T02:44:36,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 in 169 msec 2024-11-24T02:44:36,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-24T02:44:36,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, ASSIGN in 325 msec 2024-11-24T02:44:36,019 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:44:36,019 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416276019"}]},"ts":"1732416276019"} 2024-11-24T02:44:36,020 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:44:36,021 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:44:36,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-24T02:44:37,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-24T02:44:37,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:37,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ae4ba4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a653cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,013 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be35c44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,016 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:44:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,34019,1732416122189 2024-11-24T02:44:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59daede6 2024-11-24T02:44:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:44:37,021 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:44:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:44:37,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:44:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:44:37,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742348_1524 (size=1097) 2024-11-24T02:44:37,435 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T02:44:37,435 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T02:44:37,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:44:37,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE}] 2024-11-24T02:44:37,442 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE 2024-11-24T02:44:37,443 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:37,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE because future has completed 2024-11-24T02:44:37,444 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:44:37,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:37,598 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,598 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:44:37,599 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing fa18a3ad7ebd64590be10753e6ed8f58, disabling compactions & flushes 2024-11-24T02:44:37,599 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,599 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,599 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. after waiting 0 ms 2024-11-24T02:44:37,599 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,607 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:44:37,608 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,608 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for fa18a3ad7ebd64590be10753e6ed8f58: Waiting for close lock at 1732416277599Running coprocessor pre-close hooks at 1732416277599Disabling compacts and flushes for region at 1732416277599Disabling writes for close at 1732416277599Writing region close event to WAL at 1732416277600 (+1 ms)Running coprocessor post-close hooks at 1732416277608 (+8 ms)Closed at 1732416277608 2024-11-24T02:44:37,608 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: fa18a3ad7ebd64590be10753e6ed8f58 to self. 2024-11-24T02:44:37,610 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,611 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=CLOSED 2024-11-24T02:44:37,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:37,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T02:44:37,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 in 170 msec 2024-11-24T02:44:37,617 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,34251,1732416122955; forceNewPlan=false, retain=true 2024-11-24T02:44:37,767 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=OPENING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:37,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE because future has completed 2024-11-24T02:44:37,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:44:37,928 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,928 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:44:37,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:44:37,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,929 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,932 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,933 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:37,933 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName A 2024-11-24T02:44:37,934 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:37,934 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:37,935 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,935 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:37,935 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName B 2024-11-24T02:44:37,935 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:37,936 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:37,936 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,936 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T02:44:37,936 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa18a3ad7ebd64590be10753e6ed8f58 columnFamilyName C 2024-11-24T02:44:37,936 DEBUG [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:37,937 INFO [StoreOpener-fa18a3ad7ebd64590be10753e6ed8f58-1 {}] regionserver.HStore(327): Store=fa18a3ad7ebd64590be10753e6ed8f58/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:44:37,937 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,937 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,938 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,938 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,939 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,939 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,940 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:44:37,941 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,941 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened fa18a3ad7ebd64590be10753e6ed8f58; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61601129, jitterRate=-0.08207164704799652}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:44:37,942 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:37,942 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for fa18a3ad7ebd64590be10753e6ed8f58: Running coprocessor pre-open hook at 1732416277930Writing region info on filesystem at 1732416277930Initializing all the Stores at 1732416277931 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416277932 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416277932Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416277932Cleaning up temporary data from old regions at 1732416277939 (+7 ms)Running coprocessor post-open hooks at 1732416277942 (+3 ms)Region opened successfully at 1732416277942 2024-11-24T02:44:37,943 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., pid=149, masterSystemTime=1732416277921 2024-11-24T02:44:37,944 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,944 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:37,945 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:37,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:44:37,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-24T02:44:37,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 in 178 msec 2024-11-24T02:44:37,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-24T02:44:37,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, REOPEN/MOVE in 506 msec 2024-11-24T02:44:37,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-24T02:44:37,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-24T02:44:37,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-24T02:44:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:44:37,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:37,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615c30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cb6887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,957 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1281a178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c6f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,960 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff2b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21834640, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5ba127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,963 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7142a44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3e74b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110f2d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,968 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c19d620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,969 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4714171c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbb80ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44908d80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,973 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aeae6ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ae8872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4466fa48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7006fef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf38cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c65504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d369e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c3cfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,980 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103a13f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348f7dbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b067c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77316f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,984 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bec7fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebeeab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,34019,-1 for getting cluster id 2024-11-24T02:44:37,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:44:37,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3061a32-fff5-485a-b274-a9772a3f1f74' 2024-11-24T02:44:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:44:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3061a32-fff5-485a-b274-a9772a3f1f74" 2024-11-24T02:44:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ddee17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,34019,-1] 2024-11-24T02:44:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:44:37,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:37,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:44:37,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f9316c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:44:37,988 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,988 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,988 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,988 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,988 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,989 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,989 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,990 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,991 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:37,991 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:44:37,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,34251,1732416122955, seqNum=-1] 2024-11-24T02:44:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-24T02:44:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:44:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,994 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:37,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:44:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:37,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5] 2024-11-24T02:44:37,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:37,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:38,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:38,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:38,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243717e995eb984fd68511065fa48a4557_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416277998/Put/seqid=0 2024-11-24T02:44:38,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742349_1525 (size=12154) 2024-11-24T02:44:38,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416338021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416338022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416338022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416338024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732416278008, max timestamp=1732416278023Num uniques 859; 2024-11-24T02:44:38,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T02:44:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416278019, max timestamp=1732416278023Num uniques 859; 2024-11-24T02:44:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416278019, max timestamp=1732416278023Num uniques 859; 2024-11-24T02:44:38,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:38,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416338080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416338143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416338143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416338143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416338143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416338189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416338349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416338349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416338349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416338358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416338399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:38,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243717e995eb984fd68511065fa48a4557_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243717e995eb984fd68511065fa48a4557_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:38,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/cbf823c07e3d4439b56ba3bc9b33eabf, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:38,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/cbf823c07e3d4439b56ba3bc9b33eabf is 175, key is test_row_0/A:col10/1732416277998/Put/seqid=0 2024-11-24T02:44:38,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742350_1526 (size=30955) 2024-11-24T02:44:38,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416338659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416338659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416338659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416338669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:38,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416338708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:38,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:38,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/cbf823c07e3d4439b56ba3bc9b33eabf 2024-11-24T02:44:38,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/3dc453f10a58460c95aa128528c36a5a is 50, key is test_row_0/B:col10/1732416277998/Put/seqid=0 2024-11-24T02:44:38,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742351_1527 (size=12001) 2024-11-24T02:44:38,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/3dc453f10a58460c95aa128528c36a5a 2024-11-24T02:44:38,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/d26409ac69f94edba349648058742474 is 50, key is test_row_0/C:col10/1732416277998/Put/seqid=0 2024-11-24T02:44:38,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742352_1528 (size=12001) 2024-11-24T02:44:38,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:38,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:39,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416339169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416339178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:39,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:39,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:39,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:39,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416339219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:39,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:39,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/d26409ac69f94edba349648058742474 2024-11-24T02:44:39,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/cbf823c07e3d4439b56ba3bc9b33eabf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf 2024-11-24T02:44:39,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf, entries=150, sequenceid=15, filesize=30.2 K 2024-11-24T02:44:39,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/3dc453f10a58460c95aa128528c36a5a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a 2024-11-24T02:44:39,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:44:39,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/d26409ac69f94edba349648058742474 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474 2024-11-24T02:44:39,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:44:39,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for fa18a3ad7ebd64590be10753e6ed8f58 in 1309ms, sequenceid=15, compaction requested=false 2024-11-24T02:44:39,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:39,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:39,370 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:39,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:39,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:39,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243061b225b36a4f2c8103d824c051591e_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416278079/Put/seqid=0 2024-11-24T02:44:39,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742353_1529 (size=12154) 2024-11-24T02:44:39,600 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:44:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:39,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243061b225b36a4f2c8103d824c051591e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243061b225b36a4f2c8103d824c051591e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/dd7e3336ab2a4413ace40fde0e82f8b8, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:39,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/dd7e3336ab2a4413ace40fde0e82f8b8 is 175, key is test_row_0/A:col10/1732416278079/Put/seqid=0 2024-11-24T02:44:39,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742354_1530 (size=30955) 2024-11-24T02:44:39,812 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/dd7e3336ab2a4413ace40fde0e82f8b8 2024-11-24T02:44:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e428dfd49cff45928c8a83333d9ed47a is 50, key is test_row_0/B:col10/1732416278079/Put/seqid=0 2024-11-24T02:44:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742355_1531 (size=12001) 2024-11-24T02:44:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:40,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416340187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416340189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,194 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416340192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416340193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,222 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e428dfd49cff45928c8a83333d9ed47a 2024-11-24T02:44:40,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/a2a23cbb04f14e1799c0aa928e756c72 is 50, key is test_row_0/C:col10/1732416278079/Put/seqid=0 2024-11-24T02:44:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742356_1532 (size=12001) 2024-11-24T02:44:40,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/a2a23cbb04f14e1799c0aa928e756c72 2024-11-24T02:44:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/dd7e3336ab2a4413ace40fde0e82f8b8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8 2024-11-24T02:44:40,236 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8, entries=150, sequenceid=50, filesize=30.2 K 2024-11-24T02:44:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e428dfd49cff45928c8a83333d9ed47a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a 2024-11-24T02:44:40,241 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T02:44:40,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/a2a23cbb04f14e1799c0aa928e756c72 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72 2024-11-24T02:44:40,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416340240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,243 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T02:44:40,247 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=53.67 KB/54960 for fa18a3ad7ebd64590be10753e6ed8f58 in 877ms, sequenceid=50, compaction requested=false 2024-11-24T02:44:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-24T02:44:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-24T02:44:40,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-24T02:44:40,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2510 sec 2024-11-24T02:44:40,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.2600 sec 2024-11-24T02:44:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:40,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:44:40,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:40,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:40,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b77d466170c49f28f507d6880a00681_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:40,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742357_1533 (size=14594) 2024-11-24T02:44:40,313 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:40,316 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b77d466170c49f28f507d6880a00681_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b77d466170c49f28f507d6880a00681_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:40,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f5ccd6b4ecdb4fbda12f8d4de02b5518, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:40,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 is 175, key is test_row_0/A:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:40,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742358_1534 (size=39549) 2024-11-24T02:44:40,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416340329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416340330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416340332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416340438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416340439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416340439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416340648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416340648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416340649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,722 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 2024-11-24T02:44:40,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/022b4d141e984311990fe61ba7b8b19f is 50, key is test_row_0/B:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:40,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742359_1535 (size=12001) 2024-11-24T02:44:40,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/022b4d141e984311990fe61ba7b8b19f 2024-11-24T02:44:40,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9d28daaeadf14708b8bc5bedee6af2b9 is 50, key is test_row_0/C:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:40,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742360_1536 (size=12001) 2024-11-24T02:44:40,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416340959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416340959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416340959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9d28daaeadf14708b8bc5bedee6af2b9 2024-11-24T02:44:41,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 2024-11-24T02:44:41,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518, entries=200, sequenceid=63, filesize=38.6 K 2024-11-24T02:44:41,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/022b4d141e984311990fe61ba7b8b19f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f 2024-11-24T02:44:41,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f, entries=150, sequenceid=63, filesize=11.7 K 2024-11-24T02:44:41,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9d28daaeadf14708b8bc5bedee6af2b9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9 2024-11-24T02:44:41,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9, entries=150, sequenceid=63, filesize=11.7 K 2024-11-24T02:44:41,154 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-24T02:44:41,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for fa18a3ad7ebd64590be10753e6ed8f58 in 854ms, sequenceid=63, compaction requested=true 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:41,155 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:41,155 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:41,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:41,156 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:41,156 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:41,156 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=35.2 K 2024-11-24T02:44:41,156 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=99.1 K 2024-11-24T02:44:41,156 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518] 2024-11-24T02:44:41,156 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc453f10a58460c95aa128528c36a5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416277998 2024-11-24T02:44:41,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf823c07e3d4439b56ba3bc9b33eabf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416277998 2024-11-24T02:44:41,157 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e428dfd49cff45928c8a83333d9ed47a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732416278070 2024-11-24T02:44:41,157 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 022b4d141e984311990fe61ba7b8b19f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280192 2024-11-24T02:44:41,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd7e3336ab2a4413ace40fde0e82f8b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732416278070 2024-11-24T02:44:41,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ccd6b4ecdb4fbda12f8d4de02b5518, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280185 2024-11-24T02:44:41,161 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:41,162 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:41,162 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/b5ed71a4fbae4fd4ac0f527227effcaa is 50, key is test_row_0/B:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:41,170 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124dff1ed9fae564a9cb495596120b06533_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:41,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742361_1537 (size=12104) 2024-11-24T02:44:41,172 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124dff1ed9fae564a9cb495596120b06533_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:41,173 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dff1ed9fae564a9cb495596120b06533_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:41,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742362_1538 (size=4469) 2024-11-24T02:44:41,179 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/b5ed71a4fbae4fd4ac0f527227effcaa as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/b5ed71a4fbae4fd4ac0f527227effcaa 2024-11-24T02:44:41,183 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into b5ed71a4fbae4fd4ac0f527227effcaa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:41,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:41,183 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416281155; duration=0sec 2024-11-24T02:44:41,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:41,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:41,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:41,183 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:41,184 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:41,184 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:41,184 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=35.2 K 2024-11-24T02:44:41,184 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d26409ac69f94edba349648058742474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416277998 2024-11-24T02:44:41,184 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a23cbb04f14e1799c0aa928e756c72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732416278070 2024-11-24T02:44:41,184 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d28daaeadf14708b8bc5bedee6af2b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280192 2024-11-24T02:44:41,190 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:41,190 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/a2a2e4126970479aaf23d5a2f1b80172 is 50, key is test_row_0/C:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742363_1539 (size=12104) 2024-11-24T02:44:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:41,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:41,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:41,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242f521c4a689b4b85872fe5e1ccde090e_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416280331/Put/seqid=0 2024-11-24T02:44:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742364_1540 (size=14594) 2024-11-24T02:44:41,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416341483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416341485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416341486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,577 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:41,577 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7eb3f3f44ffe4121a1142b9c351fd007 is 175, key is test_row_0/A:col10/1732416280192/Put/seqid=0 2024-11-24T02:44:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742365_1541 (size=31058) 2024-11-24T02:44:41,596 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/a2a2e4126970479aaf23d5a2f1b80172 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a2e4126970479aaf23d5a2f1b80172 2024-11-24T02:44:41,600 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into a2a2e4126970479aaf23d5a2f1b80172(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:41,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:41,600 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416281155; duration=0sec 2024-11-24T02:44:41,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:41,600 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416341599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416341599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,603 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416341599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416341809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:41,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:41,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:41,884 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242f521c4a689b4b85872fe5e1ccde090e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f521c4a689b4b85872fe5e1ccde090e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:41,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4c3355e4c7614d4b9e40708d96c2d2ec, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:41,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4c3355e4c7614d4b9e40708d96c2d2ec is 175, key is test_row_0/A:col10/1732416280331/Put/seqid=0 2024-11-24T02:44:41,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742366_1542 (size=39549) 2024-11-24T02:44:41,899 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4c3355e4c7614d4b9e40708d96c2d2ec 2024-11-24T02:44:41,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/67266ac335cc488a8014a21a052628c3 is 50, key is test_row_0/B:col10/1732416280331/Put/seqid=0 2024-11-24T02:44:41,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742367_1543 (size=12001) 2024-11-24T02:44:41,983 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7eb3f3f44ffe4121a1142b9c351fd007 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007 2024-11-24T02:44:41,987 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 7eb3f3f44ffe4121a1142b9c351fd007(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:41,987 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:41,987 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416281155; duration=0sec 2024-11-24T02:44:41,987 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:41,987 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416342118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416342119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416342119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:44:42,159 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:42,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-24T02:44:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:42,161 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:42,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:42,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:42,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416342209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,211 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416342269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,271 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:42,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/67266ac335cc488a8014a21a052628c3 2024-11-24T02:44:42,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T02:44:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:42,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:42,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:42,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5fad8e73fb0b495bb1d979f0976faa0b is 50, key is test_row_0/C:col10/1732416280331/Put/seqid=0 2024-11-24T02:44:42,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742368_1544 (size=12001) 2024-11-24T02:44:42,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5fad8e73fb0b495bb1d979f0976faa0b 2024-11-24T02:44:42,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4c3355e4c7614d4b9e40708d96c2d2ec as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec 2024-11-24T02:44:42,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec, entries=200, sequenceid=84, filesize=38.6 K 2024-11-24T02:44:42,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/67266ac335cc488a8014a21a052628c3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3 2024-11-24T02:44:42,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3, entries=150, sequenceid=84, filesize=11.7 K 2024-11-24T02:44:42,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5fad8e73fb0b495bb1d979f0976faa0b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b 2024-11-24T02:44:42,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-24T02:44:42,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fa18a3ad7ebd64590be10753e6ed8f58 in 876ms, sequenceid=84, compaction requested=false 2024-11-24T02:44:42,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:42,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:42,467 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:42,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:42,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124df3b6ccfcb6b4979bd93ba2c41c49717_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416281485/Put/seqid=0 2024-11-24T02:44:42,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742369_1545 (size=12154) 2024-11-24T02:44:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:42,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:42,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416342655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416342659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416342660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416342769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416342769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416342769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:42,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:42,880 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124df3b6ccfcb6b4979bd93ba2c41c49717_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df3b6ccfcb6b4979bd93ba2c41c49717_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:42,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/69faa44c1b224aecb240c18e8fc96bc9, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:42,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/69faa44c1b224aecb240c18e8fc96bc9 is 175, key is test_row_0/A:col10/1732416281485/Put/seqid=0 2024-11-24T02:44:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742370_1546 (size=30955) 2024-11-24T02:44:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416342979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416342979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416342979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,286 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/69faa44c1b224aecb240c18e8fc96bc9 2024-11-24T02:44:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416343289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416343289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416343289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,295 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/497901615aec48e0a80945b95e36efc6 is 50, key is test_row_0/B:col10/1732416281485/Put/seqid=0 2024-11-24T02:44:43,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:43,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742371_1547 (size=12001) 2024-11-24T02:44:43,321 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/497901615aec48e0a80945b95e36efc6 2024-11-24T02:44:43,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/4ac41096444a4fb6bde6a822fa1f4108 is 50, key is test_row_0/C:col10/1732416281485/Put/seqid=0 2024-11-24T02:44:43,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742372_1548 (size=12001) 2024-11-24T02:44:43,750 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/4ac41096444a4fb6bde6a822fa1f4108 2024-11-24T02:44:43,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/69faa44c1b224aecb240c18e8fc96bc9 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9 2024-11-24T02:44:43,761 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9, entries=150, sequenceid=98, filesize=30.2 K 2024-11-24T02:44:43,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/497901615aec48e0a80945b95e36efc6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6 2024-11-24T02:44:43,767 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T02:44:43,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/4ac41096444a4fb6bde6a822fa1f4108 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108 2024-11-24T02:44:43,774 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T02:44:43,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fa18a3ad7ebd64590be10753e6ed8f58 in 1309ms, sequenceid=98, compaction requested=true 2024-11-24T02:44:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:43,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-24T02:44:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-24T02:44:43,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-24T02:44:43,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-24T02:44:43,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6210 sec 2024-11-24T02:44:43,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:43,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:43,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:43,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124004dc4aa8ad143a9a631ec1f1bdf714e_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742373_1549 (size=14594) 2024-11-24T02:44:43,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416343826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416343833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416343833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416343940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416343940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416343949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416344149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416344149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416344159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:44,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124004dc4aa8ad143a9a631ec1f1bdf714e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124004dc4aa8ad143a9a631ec1f1bdf714e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:44,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f0f22a0c28554f88babf7707a815cbfd, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:44,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f0f22a0c28554f88babf7707a815cbfd is 175, key is test_row_0/A:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:44,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742374_1550 (size=39549) 2024-11-24T02:44:44,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f0f22a0c28554f88babf7707a815cbfd 2024-11-24T02:44:44,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/090b3f2700e341d79728cdd4787fccbd is 50, key is test_row_0/B:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:44,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742375_1551 (size=12001) 2024-11-24T02:44:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T02:44:44,320 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:44,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T02:44:44,323 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:44,329 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:44,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416344459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416344469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:44,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:44,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/090b3f2700e341d79728cdd4787fccbd 2024-11-24T02:44:44,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/7f29459b230a408f9ce687717d96c893 is 50, key is test_row_0/C:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742376_1552 (size=12001) 2024-11-24T02:44:44,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:44,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:44,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416344969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416344979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416344979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:44,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:45,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:45,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:45,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/7f29459b230a408f9ce687717d96c893 2024-11-24T02:44:45,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f0f22a0c28554f88babf7707a815cbfd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd 2024-11-24T02:44:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd, entries=200, sequenceid=119, filesize=38.6 K 2024-11-24T02:44:45,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/090b3f2700e341d79728cdd4787fccbd as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd 2024-11-24T02:44:45,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd, entries=150, sequenceid=119, filesize=11.7 K 2024-11-24T02:44:45,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/7f29459b230a408f9ce687717d96c893 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893 2024-11-24T02:44:45,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893, entries=150, sequenceid=119, filesize=11.7 K 2024-11-24T02:44:45,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for fa18a3ad7ebd64590be10753e6ed8f58 in 1353ms, sequenceid=119, compaction requested=true 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:45,156 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:45,156 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:45,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:45,157 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,157 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,157 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=137.8 K 2024-11-24T02:44:45,157 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/b5ed71a4fbae4fd4ac0f527227effcaa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=47.0 K 2024-11-24T02:44:45,157 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd] 2024-11-24T02:44:45,157 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ed71a4fbae4fd4ac0f527227effcaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280192 2024-11-24T02:44:45,159 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eb3f3f44ffe4121a1142b9c351fd007, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280192 2024-11-24T02:44:45,159 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 67266ac335cc488a8014a21a052628c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732416280328 2024-11-24T02:44:45,159 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 497901615aec48e0a80945b95e36efc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732416281476 2024-11-24T02:44:45,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3355e4c7614d4b9e40708d96c2d2ec, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732416280315 2024-11-24T02:44:45,160 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 090b3f2700e341d79728cdd4787fccbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:45,160 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69faa44c1b224aecb240c18e8fc96bc9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732416281476 2024-11-24T02:44:45,162 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f22a0c28554f88babf7707a815cbfd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:45,170 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:45,171 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/0e6e7a7ed3c34729873ccdc45c7ec61a is 50, key is test_row_0/B:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:45,185 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742377_1553 (size=12241) 2024-11-24T02:44:45,210 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112492e9a3644bfc444d8589637286857c2a_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:45,213 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112492e9a3644bfc444d8589637286857c2a_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:45,213 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112492e9a3644bfc444d8589637286857c2a_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:45,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742378_1554 (size=4469) 2024-11-24T02:44:45,226 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#472 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:45,226 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/27d4e488231949199fa4fe3efd69ea12 is 175, key is test_row_0/A:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:45,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742379_1555 (size=31195) 2024-11-24T02:44:45,235 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/27d4e488231949199fa4fe3efd69ea12 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12 2024-11-24T02:44:45,240 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 27d4e488231949199fa4fe3efd69ea12(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:45,240 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:45,240 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=12, startTime=1732416285156; duration=0sec 2024-11-24T02:44:45,240 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:45,240 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:45,241 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:45,242 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:45,242 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:45,242 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,242 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a2e4126970479aaf23d5a2f1b80172, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=47.0 K 2024-11-24T02:44:45,243 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2a2e4126970479aaf23d5a2f1b80172, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732416280192 2024-11-24T02:44:45,243 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fad8e73fb0b495bb1d979f0976faa0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732416280328 2024-11-24T02:44:45,243 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac41096444a4fb6bde6a822fa1f4108, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732416281476 2024-11-24T02:44:45,243 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f29459b230a408f9ce687717d96c893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:45,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T02:44:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:45,252 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:45,255 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:45,256 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3cc544561bf04191bb67b4aacda69d4c is 50, key is test_row_0/C:col10/1732416282658/Put/seqid=0 2024-11-24T02:44:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112458ba3fc8dc504635b0993183ef1fe5a4_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416283831/Put/seqid=0 2024-11-24T02:44:45,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742380_1556 (size=12241) 2024-11-24T02:44:45,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742381_1557 (size=12204) 2024-11-24T02:44:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:45,608 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/0e6e7a7ed3c34729873ccdc45c7ec61a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/0e6e7a7ed3c34729873ccdc45c7ec61a 2024-11-24T02:44:45,615 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into 0e6e7a7ed3c34729873ccdc45c7ec61a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:45,615 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:45,615 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=12, startTime=1732416285156; duration=0sec 2024-11-24T02:44:45,615 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:45,615 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:45,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:45,667 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3cc544561bf04191bb67b4aacda69d4c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3cc544561bf04191bb67b4aacda69d4c 2024-11-24T02:44:45,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112458ba3fc8dc504635b0993183ef1fe5a4_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458ba3fc8dc504635b0993183ef1fe5a4_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:45,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/94b9f57531504fe2ad3a45d981cbf18f, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:45,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/94b9f57531504fe2ad3a45d981cbf18f is 175, key is test_row_0/A:col10/1732416283831/Put/seqid=0 2024-11-24T02:44:45,682 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into 3cc544561bf04191bb67b4aacda69d4c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:45,682 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:45,682 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=12, startTime=1732416285156; duration=0sec 2024-11-24T02:44:45,682 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:45,682 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:45,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742382_1558 (size=31005) 2024-11-24T02:44:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:46,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416346016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416346018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416346019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,103 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/94b9f57531504fe2ad3a45d981cbf18f 2024-11-24T02:44:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4cde6ba21cd64118bb7ba253958fd380 is 50, key is test_row_0/B:col10/1732416283831/Put/seqid=0 2024-11-24T02:44:46,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742383_1559 (size=12051) 2024-11-24T02:44:46,122 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4cde6ba21cd64118bb7ba253958fd380 2024-11-24T02:44:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/c0b4833d5b5c40b087fea47603739222 is 50, key is test_row_0/C:col10/1732416283831/Put/seqid=0 2024-11-24T02:44:46,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416346129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416346129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416346129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,131 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742384_1560 (size=12051) 2024-11-24T02:44:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416346229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,232 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416346289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,292 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416346339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416346339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416346339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:46,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/c0b4833d5b5c40b087fea47603739222 2024-11-24T02:44:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/94b9f57531504fe2ad3a45d981cbf18f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f 2024-11-24T02:44:46,541 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f, entries=150, sequenceid=132, filesize=30.3 K 2024-11-24T02:44:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4cde6ba21cd64118bb7ba253958fd380 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380 2024-11-24T02:44:46,545 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T02:44:46,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/c0b4833d5b5c40b087fea47603739222 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222 2024-11-24T02:44:46,550 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T02:44:46,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fa18a3ad7ebd64590be10753e6ed8f58 in 1299ms, sequenceid=132, compaction requested=false 2024-11-24T02:44:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T02:44:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T02:44:46,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T02:44:46,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2230 sec 2024-11-24T02:44:46,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2340 sec 2024-11-24T02:44:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:46,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-24T02:44:46,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:46,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:46,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:46,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:46,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:46,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:46,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248932e1697772497e9f7b3288c35df7b0_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:46,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742385_1561 (size=12304) 2024-11-24T02:44:46,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:46,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248932e1697772497e9f7b3288c35df7b0_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248932e1697772497e9f7b3288c35df7b0_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:46,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6b63026df10d444f8c140c0b1a0d9e59, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:46,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6b63026df10d444f8c140c0b1a0d9e59 is 175, key is test_row_0/A:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416346695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742386_1562 (size=31105) 2024-11-24T02:44:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416346698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416346699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416346809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416346809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416346809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:46,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416347019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416347019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416347019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6b63026df10d444f8c140c0b1a0d9e59 2024-11-24T02:44:47,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ea3d7d4d610a45b0a2fa2af2b52d54de is 50, key is test_row_0/B:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742387_1563 (size=12151) 2024-11-24T02:44:47,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416347329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416347329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416347329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ea3d7d4d610a45b0a2fa2af2b52d54de 2024-11-24T02:44:47,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9448bfde47094fa7bdfb9727de1786d8 is 50, key is test_row_0/C:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:47,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742388_1564 (size=12151) 2024-11-24T02:44:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416347839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416347849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416347849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:47,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:47,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9448bfde47094fa7bdfb9727de1786d8 2024-11-24T02:44:47,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6b63026df10d444f8c140c0b1a0d9e59 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59 2024-11-24T02:44:47,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59, entries=150, sequenceid=157, filesize=30.4 K 2024-11-24T02:44:47,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ea3d7d4d610a45b0a2fa2af2b52d54de as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de 2024-11-24T02:44:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T02:44:47,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9448bfde47094fa7bdfb9727de1786d8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8 2024-11-24T02:44:47,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8, entries=150, sequenceid=157, filesize=11.9 K 2024-11-24T02:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for fa18a3ad7ebd64590be10753e6ed8f58 in 1270ms, sequenceid=157, compaction requested=true 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:47,949 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:47,949 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:47,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,950 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:47,950 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:47,950 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:47,950 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=91.1 K 2024-11-24T02:44:47,950 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,950 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59] 2024-11-24T02:44:47,950 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:47,950 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:47,951 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:47,951 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/0e6e7a7ed3c34729873ccdc45c7ec61a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=35.6 K 2024-11-24T02:44:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27d4e488231949199fa4fe3efd69ea12, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e6e7a7ed3c34729873ccdc45c7ec61a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cde6ba21cd64118bb7ba253958fd380, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416283822 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94b9f57531504fe2ad3a45d981cbf18f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416283822 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting ea3d7d4d610a45b0a2fa2af2b52d54de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:47,951 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b63026df10d444f8c140c0b1a0d9e59, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,957 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,958 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,959 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/a17caffed45941bb85c8173d132a218a is 50, key is test_row_0/B:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,969 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124385524368e24449f86371112021a84cf_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,971 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124385524368e24449f86371112021a84cf_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,971 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124385524368e24449f86371112021a84cf_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742389_1565 (size=12493) 2024-11-24T02:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,989 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/a17caffed45941bb85c8173d132a218a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17caffed45941bb85c8173d132a218a 2024-11-24T02:44:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,992 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into a17caffed45941bb85c8173d132a218a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:47,992 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,993 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416287949; duration=0sec 2024-11-24T02:44:47,993 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:47,993 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:47,993 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,993 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:47,993 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:47,994 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:47,994 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3cc544561bf04191bb67b4aacda69d4c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=35.6 K 2024-11-24T02:44:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,994 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc544561bf04191bb67b4aacda69d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732416282654 2024-11-24T02:44:47,994 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b4833d5b5c40b087fea47603739222, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416283822 2024-11-24T02:44:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,994 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9448bfde47094fa7bdfb9727de1786d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742390_1566 (size=4469) 2024-11-24T02:44:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,010 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,011 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5cf1cc993a134285b69db89c5428331e is 50, key is test_row_0/C:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742391_1567 (size=12493) 2024-11-24T02:44:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,405 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,405 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6f7a1fb7581649729c548dd1b135f950 is 175, key is test_row_0/A:col10/1732416286650/Put/seqid=0 2024-11-24T02:44:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,421 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5cf1cc993a134285b69db89c5428331e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5cf1cc993a134285b69db89c5428331e 2024-11-24T02:44:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,426 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into 5cf1cc993a134285b69db89c5428331e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:48,426 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:48,426 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416287949; duration=0sec 2024-11-24T02:44:48,426 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:48,426 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742392_1568 (size=31447) 2024-11-24T02:44:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,438 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6f7a1fb7581649729c548dd1b135f950 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950 2024-11-24T02:44:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,444 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 6f7a1fb7581649729c548dd1b135f950(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:48,444 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:48,444 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416287949; duration=0sec 2024-11-24T02:44:48,444 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:48,444 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T02:44:48,489 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T02:44:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,492 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T02:44:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T02:44:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T02:44:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:48,648 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T02:44:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5f5521909c14303b8e4731b001ddc41_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416286693/Put/seqid=0 2024-11-24T02:44:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742393_1569 (size=9814) 2024-11-24T02:44:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T02:44:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:48,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416288893, max timestamp=1732416288912Num uniques 859; 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416288893, max timestamp=1732416288912Num uniques 859; 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416288893, max timestamp=1732416288912Num uniques 859; 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:48,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416348938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:48,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416348940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:48,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:48,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416348942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:48,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416349049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416349049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416349059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:49,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5f5521909c14303b8e4731b001ddc41_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f5521909c14303b8e4731b001ddc41_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:49,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/2fb1114a1d45411e80aa1bb72b63c3c5, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/2fb1114a1d45411e80aa1bb72b63c3c5 is 175, key is test_row_0/A:col10/1732416286693/Put/seqid=0 2024-11-24T02:44:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742394_1570 (size=22461) 2024-11-24T02:44:49,098 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/2fb1114a1d45411e80aa1bb72b63c3c5 2024-11-24T02:44:49,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/dfe5d57b5fdd4fae96600657c8cd648f is 50, key is test_row_0/B:col10/1732416286693/Put/seqid=0 2024-11-24T02:44:49,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742395_1571 (size=9757) 2024-11-24T02:44:49,112 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/dfe5d57b5fdd4fae96600657c8cd648f 2024-11-24T02:44:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T02:44:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/86165ff8b7e6470da334cc001c4269aa is 50, key is test_row_0/C:col10/1732416286693/Put/seqid=0 2024-11-24T02:44:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742396_1572 (size=9757) 2024-11-24T02:44:49,125 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/86165ff8b7e6470da334cc001c4269aa 2024-11-24T02:44:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/2fb1114a1d45411e80aa1bb72b63c3c5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5 2024-11-24T02:44:49,139 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5, entries=100, sequenceid=169, filesize=21.9 K 2024-11-24T02:44:49,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/dfe5d57b5fdd4fae96600657c8cd648f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f 2024-11-24T02:44:49,144 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f, entries=100, sequenceid=169, filesize=9.5 K 2024-11-24T02:44:49,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/86165ff8b7e6470da334cc001c4269aa as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa 2024-11-24T02:44:49,149 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa, entries=100, sequenceid=169, filesize=9.5 K 2024-11-24T02:44:49,150 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for fa18a3ad7ebd64590be10753e6ed8f58 in 503ms, sequenceid=169, compaction requested=false 2024-11-24T02:44:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T02:44:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T02:44:49,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T02:44:49,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-11-24T02:44:49,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 665 msec 2024-11-24T02:44:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:49,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:49,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T02:44:49,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a104291dfa5a4cf29ef95845eb236b76_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416349269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416349272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416349274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742397_1573 (size=14794) 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:49,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a104291dfa5a4cf29ef95845eb236b76_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a104291dfa5a4cf29ef95845eb236b76_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:49,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f400a5f1b7e847db96d0c53846b49a47, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f400a5f1b7e847db96d0c53846b49a47 is 175, key is test_row_0/A:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742398_1574 (size=39749) 2024-11-24T02:44:49,304 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f400a5f1b7e847db96d0c53846b49a47 2024-11-24T02:44:49,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/70be2bb67aca4c65a9b63cc14aecdda5 is 50, key is test_row_0/B:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742399_1575 (size=12151) 2024-11-24T02:44:49,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/70be2bb67aca4c65a9b63cc14aecdda5 2024-11-24T02:44:49,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/633120ef79b14e7699c421955922d10f is 50, key is test_row_0/C:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742400_1576 (size=12151) 2024-11-24T02:44:49,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/633120ef79b14e7699c421955922d10f 2024-11-24T02:44:49,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/f400a5f1b7e847db96d0c53846b49a47 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47 2024-11-24T02:44:49,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47, entries=200, sequenceid=207, filesize=38.8 K 2024-11-24T02:44:49,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/70be2bb67aca4c65a9b63cc14aecdda5 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5 2024-11-24T02:44:49,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T02:44:49,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/633120ef79b14e7699c421955922d10f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f 2024-11-24T02:44:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416349389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416349389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f, entries=150, sequenceid=207, filesize=11.9 K 2024-11-24T02:44:49,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for fa18a3ad7ebd64590be10753e6ed8f58 in 135ms, sequenceid=207, compaction requested=true 2024-11-24T02:44:49,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:49,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:49,395 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:49,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:49,396 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:49,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:49,396 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:49,396 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:49,397 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,397 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=91.5 K 2024-11-24T02:44:49,397 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,397 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47] 2024-11-24T02:44:49,397 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f7a1fb7581649729c548dd1b135f950, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:49,397 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb1114a1d45411e80aa1bb72b63c3c5, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416286693 2024-11-24T02:44:49,399 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting f400a5f1b7e847db96d0c53846b49a47, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:49,403 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:49,403 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:49,403 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,403 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17caffed45941bb85c8173d132a218a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=33.6 K 2024-11-24T02:44:49,404 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a17caffed45941bb85c8173d132a218a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:49,405 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe5d57b5fdd4fae96600657c8cd648f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416286693 2024-11-24T02:44:49,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 70be2bb67aca4c65a9b63cc14aecdda5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:49,413 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,426 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:49,426 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4451d629e0524fa480e85c78dcb0e61e is 50, key is test_row_0/B:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,428 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411244e1326777ab34eff97b0105f3917eb90_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,430 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411244e1326777ab34eff97b0105f3917eb90_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,430 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244e1326777ab34eff97b0105f3917eb90_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:49,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742402_1578 (size=12595) 2024-11-24T02:44:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742401_1577 (size=4469) 2024-11-24T02:44:49,478 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#489 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:49,479 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7eb5e6d374524b95ae101bef0bb45a71 is 175, key is test_row_0/A:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,483 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4451d629e0524fa480e85c78dcb0e61e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4451d629e0524fa480e85c78dcb0e61e 2024-11-24T02:44:49,491 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into 4451d629e0524fa480e85c78dcb0e61e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:49,491 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:49,491 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416289396; duration=0sec 2024-11-24T02:44:49,492 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:49,492 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:49,492 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:49,493 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:49,493 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:49,494 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,494 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5cf1cc993a134285b69db89c5428331e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=33.6 K 2024-11-24T02:44:49,494 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf1cc993a134285b69db89c5428331e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732416286005 2024-11-24T02:44:49,494 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 86165ff8b7e6470da334cc001c4269aa, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416286693 2024-11-24T02:44:49,495 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 633120ef79b14e7699c421955922d10f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:49,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742403_1579 (size=31549) 2024-11-24T02:44:49,534 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7eb5e6d374524b95ae101bef0bb45a71 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71 2024-11-24T02:44:49,542 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 7eb5e6d374524b95ae101bef0bb45a71(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:49,542 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:49,542 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416289395; duration=0sec 2024-11-24T02:44:49,542 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:49,542 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:49,544 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:49,544 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/6b02ededb4534349b49a280106d2a8a8 is 50, key is test_row_0/C:col10/1732416288927/Put/seqid=0 2024-11-24T02:44:49,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742404_1580 (size=12595) 2024-11-24T02:44:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:49,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:49,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:49,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242252fdfeaa9c4cf69b88c645ac66290b_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416289582/Put/seqid=0 2024-11-24T02:44:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742405_1581 (size=12304) 2024-11-24T02:44:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T02:44:49,629 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:49,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T02:44:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:49,631 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:49,631 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:49,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:49,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416289598, max timestamp=1732416289662Num uniques 859; 2024-11-24T02:44:49,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416289598, max timestamp=1732416289662Num uniques 859; 2024-11-24T02:44:49,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:49,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:49,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732416289598, max timestamp=1732416289662Num uniques 859; 2024-11-24T02:44:49,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T02:44:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416349702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416349702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416349703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:49,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:49,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:49,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416349824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416349824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416349824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:49,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:49,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:49,963 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/6b02ededb4534349b49a280106d2a8a8 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/6b02ededb4534349b49a280106d2a8a8 2024-11-24T02:44:49,967 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into 6b02ededb4534349b49a280106d2a8a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:49,967 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:49,967 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416289396; duration=0sec 2024-11-24T02:44:49,967 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:49,967 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:50,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:50,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242252fdfeaa9c4cf69b88c645ac66290b_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242252fdfeaa9c4cf69b88c645ac66290b_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:50,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7171bf64f55c486f8b4db6469dabfbba, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:50,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7171bf64f55c486f8b4db6469dabfbba is 175, key is test_row_0/A:col10/1732416289582/Put/seqid=0 2024-11-24T02:44:50,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742406_1582 (size=31105) 2024-11-24T02:44:50,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416350029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416350039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416350039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,050 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:50,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:50,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416350339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416350348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416350359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:50,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:50,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:50,425 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7171bf64f55c486f8b4db6469dabfbba 2024-11-24T02:44:50,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/f0adf050f5ac471baf88ae7465728b97 is 50, key is test_row_0/B:col10/1732416289582/Put/seqid=0 2024-11-24T02:44:50,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742407_1583 (size=12151) 2024-11-24T02:44:50,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/f0adf050f5ac471baf88ae7465728b97 2024-11-24T02:44:50,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1d450e3af6ff4339ab9a14a5d04ffbb1 is 50, key is test_row_0/C:col10/1732416289582/Put/seqid=0 2024-11-24T02:44:50,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742408_1584 (size=12151) 2024-11-24T02:44:50,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1d450e3af6ff4339ab9a14a5d04ffbb1 2024-11-24T02:44:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/7171bf64f55c486f8b4db6469dabfbba as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba 2024-11-24T02:44:50,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba, entries=150, sequenceid=219, filesize=30.4 K 2024-11-24T02:44:50,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/f0adf050f5ac471baf88ae7465728b97 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97 2024-11-24T02:44:50,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T02:44:50,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1d450e3af6ff4339ab9a14a5d04ffbb1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1 2024-11-24T02:44:50,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T02:44:50,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for fa18a3ad7ebd64590be10753e6ed8f58 in 902ms, sequenceid=219, compaction requested=false 2024-11-24T02:44:50,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:50,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:44:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:50,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:50,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:50,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112451d9241d3b244aa999eea3d4d18ee8ba_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416289683/Put/seqid=0 2024-11-24T02:44:50,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742409_1585 (size=12304) 2024-11-24T02:44:50,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:50,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416350868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416350869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416350872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:50,959 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112451d9241d3b244aa999eea3d4d18ee8ba_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451d9241d3b244aa999eea3d4d18ee8ba_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:50,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3b530ed64d29431a9392016b033a93fc, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:50,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3b530ed64d29431a9392016b033a93fc is 175, key is test_row_0/A:col10/1732416289683/Put/seqid=0 2024-11-24T02:44:50,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742410_1586 (size=31105) 2024-11-24T02:44:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416350979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416350979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:50,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416351189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416351189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:51,363 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3b530ed64d29431a9392016b033a93fc 2024-11-24T02:44:51,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 is 50, key is test_row_0/B:col10/1732416289683/Put/seqid=0 2024-11-24T02:44:51,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742411_1587 (size=12151) 2024-11-24T02:44:51,377 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 2024-11-24T02:44:51,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9bb0b7725dcb489db30f971bc37274ae is 50, key is test_row_0/C:col10/1732416289683/Put/seqid=0 2024-11-24T02:44:51,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742412_1588 (size=12151) 2024-11-24T02:44:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416351499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416351499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:51,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:51,796 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9bb0b7725dcb489db30f971bc37274ae 2024-11-24T02:44:51,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3b530ed64d29431a9392016b033a93fc as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc 2024-11-24T02:44:51,801 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc, entries=150, sequenceid=257, filesize=30.4 K 2024-11-24T02:44:51,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 2024-11-24T02:44:51,804 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66, entries=150, sequenceid=257, filesize=11.9 K 2024-11-24T02:44:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9bb0b7725dcb489db30f971bc37274ae as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae 2024-11-24T02:44:51,807 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae, entries=150, sequenceid=257, filesize=11.9 K 2024-11-24T02:44:51,808 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for fa18a3ad7ebd64590be10753e6ed8f58 in 1261ms, sequenceid=257, compaction requested=true 2024-11-24T02:44:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:51,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T02:44:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T02:44:51,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T02:44:51,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-24T02:44:51,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1820 sec 2024-11-24T02:44:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:51,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:51,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:51,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c8736417fee94f2e98c27b8424904e79_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742413_1589 (size=14994) 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416291938, max timestamp=1732416291977Num uniques 859; 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416291938, max timestamp=1732416291977Num uniques 859; 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416291938, max timestamp=1732416291977Num uniques 859; 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:51,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416352039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416292034, max timestamp=1732416292044Num uniques 1018; 2024-11-24T02:44:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:44:52,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416292034, max timestamp=1732416292044Num uniques 1018; 2024-11-24T02:44:52,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732416292034, max timestamp=1732416292044Num uniques 1018; 2024-11-24T02:44:52,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:44:52,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T02:44:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416352094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416352101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416352149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416352208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416352208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:52,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c8736417fee94f2e98c27b8424904e79_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c8736417fee94f2e98c27b8424904e79_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:52,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/8c595f6ea32a4661805108965221ccca, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:52,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/8c595f6ea32a4661805108965221ccca is 175, key is test_row_0/A:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742414_1590 (size=39949) 2024-11-24T02:44:52,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416352359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416352419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416352419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416352669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/8c595f6ea32a4661805108965221ccca 2024-11-24T02:44:52,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/fc646943a75a41629003d1f8a295c312 is 50, key is test_row_0/B:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416352739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416352739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:52,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:52,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742415_1591 (size=12301) 2024-11-24T02:44:52,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/fc646943a75a41629003d1f8a295c312 2024-11-24T02:44:52,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/77d8f721c88048308138d35eef1c6a2e is 50, key is test_row_0/C:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742416_1592 (size=12301) 2024-11-24T02:44:52,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/77d8f721c88048308138d35eef1c6a2e 2024-11-24T02:44:52,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/8c595f6ea32a4661805108965221ccca as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca 2024-11-24T02:44:52,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca, entries=200, sequenceid=267, filesize=39.0 K 2024-11-24T02:44:52,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/fc646943a75a41629003d1f8a295c312 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312 2024-11-24T02:44:52,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T02:44:52,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/77d8f721c88048308138d35eef1c6a2e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e 2024-11-24T02:44:52,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T02:44:52,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for fa18a3ad7ebd64590be10753e6ed8f58 in 975ms, sequenceid=267, compaction requested=true 2024-11-24T02:44:52,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:52,857 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:52,857 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:52,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:52,858 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133708 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:52,858 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:52,858 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:52,858 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:52,858 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:52,858 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=130.6 K 2024-11-24T02:44:52,858 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:52,858 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:52,858 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4451d629e0524fa480e85c78dcb0e61e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=48.0 K 2024-11-24T02:44:52,858 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca] 2024-11-24T02:44:52,859 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4451d629e0524fa480e85c78dcb0e61e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:52,859 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eb5e6d374524b95ae101bef0bb45a71, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:52,860 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f0adf050f5ac471baf88ae7465728b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732416289265 2024-11-24T02:44:52,861 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7171bf64f55c486f8b4db6469dabfbba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732416289265 2024-11-24T02:44:52,861 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4ca9a3cfd5492bb191a87b9c3f9f66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732416289683 2024-11-24T02:44:52,862 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b530ed64d29431a9392016b033a93fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732416289683 2024-11-24T02:44:52,862 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fc646943a75a41629003d1f8a295c312, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:52,862 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c595f6ea32a4661805108965221ccca, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:52,873 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:52,895 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:52,895 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/1bc93c6f97b740f0930aa7b61ee88021 is 50, key is test_row_0/B:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,901 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124c7c7032751a244bfa5fbc3a9a1044a74_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:52,904 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124c7c7032751a244bfa5fbc3a9a1044a74_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:52,904 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c7c7032751a244bfa5fbc3a9a1044a74_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:52,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742417_1593 (size=12881) 2024-11-24T02:44:52,924 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/1bc93c6f97b740f0930aa7b61ee88021 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/1bc93c6f97b740f0930aa7b61ee88021 2024-11-24T02:44:52,933 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into 1bc93c6f97b740f0930aa7b61ee88021(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:52,933 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:52,933 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=12, startTime=1732416292857; duration=0sec 2024-11-24T02:44:52,933 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:52,933 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:52,933 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:44:52,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742418_1594 (size=4469) 2024-11-24T02:44:52,936 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:44:52,936 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:52,936 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:52,936 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/6b02ededb4534349b49a280106d2a8a8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=48.0 K 2024-11-24T02:44:52,936 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b02ededb4534349b49a280106d2a8a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732416288927 2024-11-24T02:44:52,937 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d450e3af6ff4339ab9a14a5d04ffbb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732416289265 2024-11-24T02:44:52,937 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb0b7725dcb489db30f971bc37274ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732416289683 2024-11-24T02:44:52,938 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d8f721c88048308138d35eef1c6a2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:52,938 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#501 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:52,939 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/a83d3e58bc5543d080980c2e0ddce0ce is 175, key is test_row_0/A:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,959 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#503 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:52,959 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/d89e0eed0b944e5fa35d9e657b900d29 is 50, key is test_row_0/C:col10/1732416290871/Put/seqid=0 2024-11-24T02:44:52,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742420_1596 (size=12881) 2024-11-24T02:44:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742419_1595 (size=31835) 2024-11-24T02:44:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:53,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:53,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f4ba564af24742fc8f966f60a895bfcc_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416292078/Put/seqid=0 2024-11-24T02:44:53,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742421_1597 (size=14994) 2024-11-24T02:44:53,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:53,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f4ba564af24742fc8f966f60a895bfcc_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f4ba564af24742fc8f966f60a895bfcc_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:53,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/9332750baed64529b7db8f9d5cc4809b, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/9332750baed64529b7db8f9d5cc4809b is 175, key is test_row_0/A:col10/1732416292078/Put/seqid=0 2024-11-24T02:44:53,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742422_1598 (size=39949) 2024-11-24T02:44:53,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/9332750baed64529b7db8f9d5cc4809b 2024-11-24T02:44:53,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416353223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:53,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4429c816b7304154bb7f404679c8af4e is 50, key is test_row_0/B:col10/1732416292078/Put/seqid=0 2024-11-24T02:44:53,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416353249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:53,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416353259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:53,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742423_1599 (size=12301) 2024-11-24T02:44:53,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416353339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:53,400 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/d89e0eed0b944e5fa35d9e657b900d29 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d89e0eed0b944e5fa35d9e657b900d29 2024-11-24T02:44:53,404 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/a83d3e58bc5543d080980c2e0ddce0ce as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce 2024-11-24T02:44:53,409 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into d89e0eed0b944e5fa35d9e657b900d29(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:53,409 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:53,409 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=12, startTime=1732416292857; duration=0sec 2024-11-24T02:44:53,409 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:53,409 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:53,411 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into a83d3e58bc5543d080980c2e0ddce0ce(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:53,411 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:53,411 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=12, startTime=1732416292856; duration=0sec 2024-11-24T02:44:53,411 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:53,411 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:53,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416353551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:53,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:53,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4429c816b7304154bb7f404679c8af4e 2024-11-24T02:44:53,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/553e7e9609bc49d4a9a9f98cae750e40 is 50, key is test_row_0/C:col10/1732416292078/Put/seqid=0 2024-11-24T02:44:53,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742424_1600 (size=12301) 2024-11-24T02:44:53,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/553e7e9609bc49d4a9a9f98cae750e40 2024-11-24T02:44:53,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/9332750baed64529b7db8f9d5cc4809b as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b 2024-11-24T02:44:53,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b, entries=200, sequenceid=322, filesize=39.0 K 2024-11-24T02:44:53,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/4429c816b7304154bb7f404679c8af4e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e 2024-11-24T02:44:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:44:53,809 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:53,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T02:44:53,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:44:53,814 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:53,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:53,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e, entries=150, sequenceid=322, filesize=12.0 K 2024-11-24T02:44:53,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/553e7e9609bc49d4a9a9f98cae750e40 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40 2024-11-24T02:44:53,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40, entries=150, sequenceid=322, filesize=12.0 K 2024-11-24T02:44:53,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for fa18a3ad7ebd64590be10753e6ed8f58 in 640ms, sequenceid=322, compaction requested=false 2024-11-24T02:44:53,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:53,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T02:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:53,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:53,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:53,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248cef208ea95d4c838bc87cc5d407d641_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:53,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742425_1601 (size=14994) 2024-11-24T02:44:53,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:53,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248cef208ea95d4c838bc87cc5d407d641_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248cef208ea95d4c838bc87cc5d407d641_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:53,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6ce87c5887fe4de0bdca01edd31cc1d6, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6ce87c5887fe4de0bdca01edd31cc1d6 is 175, key is test_row_0/A:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:53,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742426_1602 (size=39949) 2024-11-24T02:44:53,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6ce87c5887fe4de0bdca01edd31cc1d6 2024-11-24T02:44:53,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/c43228cb92354e03bc3ac040995fbee6 is 50, key is test_row_0/B:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:53,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742427_1603 (size=12301) 2024-11-24T02:44:53,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/c43228cb92354e03bc3ac040995fbee6 2024-11-24T02:44:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3591b2ca008b47a0a6d3e1eb71f21b51 is 50, key is test_row_0/C:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:44:53,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742428_1604 (size=12301) 2024-11-24T02:44:53,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3591b2ca008b47a0a6d3e1eb71f21b51 2024-11-24T02:44:53,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/6ce87c5887fe4de0bdca01edd31cc1d6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6 2024-11-24T02:44:53,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6, entries=200, sequenceid=334, filesize=39.0 K 2024-11-24T02:44:53,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/c43228cb92354e03bc3ac040995fbee6 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6 2024-11-24T02:44:53,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6, entries=150, sequenceid=334, filesize=12.0 K 2024-11-24T02:44:53,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3591b2ca008b47a0a6d3e1eb71f21b51 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416293917, max timestamp=1732416293946Num uniques 859; 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416293917, max timestamp=1732416293946Num uniques 859; 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416293917, max timestamp=1732416293946Num uniques 859; 2024-11-24T02:44:53,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51, entries=150, sequenceid=334, filesize=12.0 K 2024-11-24T02:44:53,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for fa18a3ad7ebd64590be10753e6ed8f58 in 81ms, sequenceid=334, compaction requested=true 2024-11-24T02:44:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:53,953 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:53,953 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:53,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111733 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:53,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:53,954 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:53,954 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=109.1 K 2024-11-24T02:44:53,954 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:53,954 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6] 2024-11-24T02:44:53,955 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:53,955 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:53,955 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:53,955 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/1bc93c6f97b740f0930aa7b61ee88021, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.6 K 2024-11-24T02:44:53,955 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc93c6f97b740f0930aa7b61ee88021, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:53,955 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting a83d3e58bc5543d080980c2e0ddce0ce, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:53,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 4429c816b7304154bb7f404679c8af4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416292078 2024-11-24T02:44:53,956 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9332750baed64529b7db8f9d5cc4809b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416292078 2024-11-24T02:44:53,956 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting c43228cb92354e03bc3ac040995fbee6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:53,956 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ce87c5887fe4de0bdca01edd31cc1d6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:53,970 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:53,971 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/f06735f8a52f4814b9e4d21b09536795 is 50, key is test_row_0/B:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:53,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T02:44:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:53,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:53,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:53,971 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-24T02:44:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:53,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:53,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:53,972 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,981 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112489539af70d83404fbbce0cbdfd64335e_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,983 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112489539af70d83404fbbce0cbdfd64335e_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,983 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112489539af70d83404fbbce0cbdfd64335e_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:53,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242ca31e6d284c4c6bbf59fb963b937c05_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416293968/Put/seqid=0 2024-11-24T02:44:54,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742429_1605 (size=12983) 2024-11-24T02:44:54,015 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/f06735f8a52f4814b9e4d21b09536795 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f06735f8a52f4814b9e4d21b09536795 2024-11-24T02:44:54,021 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into f06735f8a52f4814b9e4d21b09536795(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:54,021 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:54,021 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416293953; duration=0sec 2024-11-24T02:44:54,022 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:54,022 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:54,022 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:54,023 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:54,023 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:54,023 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,023 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d89e0eed0b944e5fa35d9e657b900d29, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.6 K 2024-11-24T02:44:54,028 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting d89e0eed0b944e5fa35d9e657b900d29, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732416290862 2024-11-24T02:44:54,029 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 553e7e9609bc49d4a9a9f98cae750e40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1732416292078 2024-11-24T02:44:54,029 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3591b2ca008b47a0a6d3e1eb71f21b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:54,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742430_1606 (size=4469) 2024-11-24T02:44:54,032 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#511 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:54,032 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/1479bda27b4c4c10be6787142797824f is 175, key is test_row_0/A:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742431_1607 (size=14994) 2024-11-24T02:44:54,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:54,046 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:54,046 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3041a15095de4f2daa11538f54abfd6a is 50, key is test_row_0/C:col10/1732416293211/Put/seqid=0 2024-11-24T02:44:54,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242ca31e6d284c4c6bbf59fb963b937c05_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ca31e6d284c4c6bbf59fb963b937c05_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:54,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/d2aeaa4b9ec44470814aae2ef8b5cd61, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:54,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/d2aeaa4b9ec44470814aae2ef8b5cd61 is 175, key is test_row_0/A:col10/1732416293968/Put/seqid=0 2024-11-24T02:44:54,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742433_1609 (size=12983) 2024-11-24T02:44:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742432_1608 (size=31937) 2024-11-24T02:44:54,094 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3041a15095de4f2daa11538f54abfd6a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3041a15095de4f2daa11538f54abfd6a 2024-11-24T02:44:54,096 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/1479bda27b4c4c10be6787142797824f as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294008, max timestamp=1732416294095Num uniques 859; 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294008, max timestamp=1732416294095Num uniques 859; 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294008, max timestamp=1732416294095Num uniques 859; 2024-11-24T02:44:54,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,102 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into 3041a15095de4f2daa11538f54abfd6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:54,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:54,102 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416293953; duration=0sec 2024-11-24T02:44:54,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:54,102 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:54,106 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 1479bda27b4c4c10be6787142797824f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:54,106 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:54,106 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416293953; duration=0sec 2024-11-24T02:44:54,106 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:54,106 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:54,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742434_1610 (size=39949) 2024-11-24T02:44:54,117 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/d2aeaa4b9ec44470814aae2ef8b5cd61 2024-11-24T02:44:54,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:44:54,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/15b619f1bb5c4c5cb21153d0b52bdff1 is 50, key is test_row_0/B:col10/1732416293968/Put/seqid=0 2024-11-24T02:44:54,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742435_1611 (size=12301) 2024-11-24T02:44:54,182 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/15b619f1bb5c4c5cb21153d0b52bdff1 2024-11-24T02:44:54,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/f11d180736764d27aa3c0d9ecf3395bb is 50, key is test_row_0/C:col10/1732416293968/Put/seqid=0 2024-11-24T02:44:54,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742436_1612 (size=12301) 2024-11-24T02:44:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/f11d180736764d27aa3c0d9ecf3395bb 2024-11-24T02:44:54,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/d2aeaa4b9ec44470814aae2ef8b5cd61 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61 2024-11-24T02:44:54,248 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61, entries=200, sequenceid=359, filesize=39.0 K 2024-11-24T02:44:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/15b619f1bb5c4c5cb21153d0b52bdff1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1 2024-11-24T02:44:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416354247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,256 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/f11d180736764d27aa3c0d9ecf3395bb as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb 2024-11-24T02:44:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416354259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,266 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:44:54,266 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,268 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for fa18a3ad7ebd64590be10753e6ed8f58 in 297ms, sequenceid=359, compaction requested=false 2024-11-24T02:44:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T02:44:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T02:44:54,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T02:44:54,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 455 msec 2024-11-24T02:44:54,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 461 msec 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732416294203, max timestamp=1732416294282Num uniques 1018; 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732416294203, max timestamp=1732416294282Num uniques 1018; 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732416294203, max timestamp=1732416294282Num uniques 1018; 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T02:44:54,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T02:44:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:54,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-24T02:44:54,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:54,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:54,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:54,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:54,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:54,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:54,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2384e3914bf4159a63626de9ff10be9_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:54,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742437_1613 (size=12454) 2024-11-24T02:44:54,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294354, max timestamp=1732416294390Num uniques 859; 2024-11-24T02:44:54,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294354, max timestamp=1732416294390Num uniques 859; 2024-11-24T02:44:54,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732416294354, max timestamp=1732416294390Num uniques 859; 2024-11-24T02:44:54,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,396 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/7c69a60bd8f6:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T02:44:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:44:54,439 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:54,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:44:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T02:44:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:54,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:44:54,444 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:44:54,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:44:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416354462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416354465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416354580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416354580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:54,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:54,765 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:54,770 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2384e3914bf4159a63626de9ff10be9_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2384e3914bf4159a63626de9ff10be9_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:54,771 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/45c9a13598b3400e864ad91523170ca1, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:54,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/45c9a13598b3400e864ad91523170ca1 is 175, key is test_row_0/A:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416354791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416354791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742438_1614 (size=31255) 2024-11-24T02:44:54,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:54,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:54,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416355098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:55,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416355098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:55,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/45c9a13598b3400e864ad91523170ca1 2024-11-24T02:44:55,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/da6421374ca64ab7bb9b78df7a65b3ce is 50, key is test_row_0/B:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:55,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742439_1615 (size=12301) 2024-11-24T02:44:55,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416355612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:55,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416355619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:55,623 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:55,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/da6421374ca64ab7bb9b78df7a65b3ce 2024-11-24T02:44:55,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5b2ddc348d3c457a9b371a0a35b11436 is 50, key is test_row_0/C:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:55,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742440_1616 (size=12301) 2024-11-24T02:44:55,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:55,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:56,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:56,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:44:56,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5b2ddc348d3c457a9b371a0a35b11436 2024-11-24T02:44:56,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/45c9a13598b3400e864ad91523170ca1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1 2024-11-24T02:44:56,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1, entries=150, sequenceid=404, filesize=30.5 K 2024-11-24T02:44:56,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/da6421374ca64ab7bb9b78df7a65b3ce as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce 2024-11-24T02:44:56,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce, entries=150, sequenceid=404, filesize=12.0 K 2024-11-24T02:44:56,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/5b2ddc348d3c457a9b371a0a35b11436 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436 2024-11-24T02:44:56,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436, entries=150, sequenceid=404, filesize=12.0 K 2024-11-24T02:44:56,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for fa18a3ad7ebd64590be10753e6ed8f58 in 1775ms, sequenceid=404, compaction requested=true 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:56,078 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:56,078 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:56,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:56,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:56,079 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:56,079 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:56,079 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,080 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f06735f8a52f4814b9e4d21b09536795, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.7 K 2024-11-24T02:44:56,080 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:56,080 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:56,080 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,080 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=100.7 K 2024-11-24T02:44:56,080 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,080 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1] 2024-11-24T02:44:56,080 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f06735f8a52f4814b9e4d21b09536795, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:56,081 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1479bda27b4c4c10be6787142797824f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:56,081 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b619f1bb5c4c5cb21153d0b52bdff1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416293958 2024-11-24T02:44:56,081 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2aeaa4b9ec44470814aae2ef8b5cd61, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416293954 2024-11-24T02:44:56,081 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting da6421374ca64ab7bb9b78df7a65b3ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:56,081 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c9a13598b3400e864ad91523170ca1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:56,109 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:56,110 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/a17e772a0fef4756aa659808141305f1 is 50, key is test_row_0/B:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:56,124 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:56,134 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124313389e8fb854d56926e87dbd3c55ae3_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:56,136 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124313389e8fb854d56926e87dbd3c55ae3_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:56,136 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124313389e8fb854d56926e87dbd3c55ae3_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:56,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,166 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:56,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:56,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742441_1617 (size=13085) 2024-11-24T02:44:56,173 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/a17e772a0fef4756aa659808141305f1 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17e772a0fef4756aa659808141305f1 2024-11-24T02:44:56,179 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into a17e772a0fef4756aa659808141305f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:56,179 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:56,179 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416296078; duration=0sec 2024-11-24T02:44:56,179 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:56,179 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:56,179 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:56,180 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:56,180 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:56,180 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:56,180 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3041a15095de4f2daa11538f54abfd6a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.7 K 2024-11-24T02:44:56,180 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 3041a15095de4f2daa11538f54abfd6a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732416293196 2024-11-24T02:44:56,181 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting f11d180736764d27aa3c0d9ecf3395bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416293958 2024-11-24T02:44:56,182 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b2ddc348d3c457a9b371a0a35b11436, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:56,211 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:56,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246404e5b7d6a04a479ef0f4ed60157baf_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416294447/Put/seqid=0 2024-11-24T02:44:56,212 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9be072c3abfb416f84868106c07ee10d is 50, key is test_row_0/C:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:56,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742442_1618 (size=4469) 2024-11-24T02:44:56,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742444_1620 (size=12454) 2024-11-24T02:44:56,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742443_1619 (size=13085) 2024-11-24T02:44:56,246 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/9be072c3abfb416f84868106c07ee10d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9be072c3abfb416f84868106c07ee10d 2024-11-24T02:44:56,251 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into 9be072c3abfb416f84868106c07ee10d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:56,251 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:56,251 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416296078; duration=0sec 2024-11-24T02:44:56,251 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:56,251 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:56,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. as already flushing 2024-11-24T02:44:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416356306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416356309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52686 deadline: 1732416356329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,331 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416356419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416356419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:56,615 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#520 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:56,615 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/bab94dac511e4a788eae20fa4cb8c567 is 175, key is test_row_0/A:col10/1732416294290/Put/seqid=0 2024-11-24T02:44:56,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742445_1621 (size=32039) 2024-11-24T02:44:56,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:56,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52672 deadline: 1732416356629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52660 deadline: 1732416356629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416356629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,631 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_1' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416356629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,631 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,34251,1732416122955 for 'test_row_0' in fa18a3ad7ebd64590be10753e6ed8f58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,632 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246404e5b7d6a04a479ef0f4ed60157baf_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246404e5b7d6a04a479ef0f4ed60157baf_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:56,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/452bc2f541674e37a15b6246a29ab22e, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/452bc2f541674e37a15b6246a29ab22e is 175, key is test_row_0/A:col10/1732416294447/Put/seqid=0 2024-11-24T02:44:56,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742446_1622 (size=31255) 2024-11-24T02:44:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416356939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416356939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:56,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:56,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,029 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/bab94dac511e4a788eae20fa4cb8c567 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567 2024-11-24T02:44:57,034 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into bab94dac511e4a788eae20fa4cb8c567(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:57,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:57,034 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416296078; duration=0sec 2024-11-24T02:44:57,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:57,034 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:57,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=441, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/452bc2f541674e37a15b6246a29ab22e 2024-11-24T02:44:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/8c2f55db21aa47419e60e05126aac4fe is 50, key is test_row_0/B:col10/1732416294447/Put/seqid=0 2024-11-24T02:44:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742447_1623 (size=12301) 2024-11-24T02:44:57,069 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/8c2f55db21aa47419e60e05126aac4fe 2024-11-24T02:44:57,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/197f1a03d4e64021a06e794cf6d9eb10 is 50, key is test_row_0/C:col10/1732416294447/Put/seqid=0 2024-11-24T02:44:57,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742448_1624 (size=12301) 2024-11-24T02:44:57,111 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/197f1a03d4e64021a06e794cf6d9eb10 2024-11-24T02:44:57,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/452bc2f541674e37a15b6246a29ab22e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e 2024-11-24T02:44:57,122 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e, entries=150, sequenceid=441, filesize=30.5 K 2024-11-24T02:44:57,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/8c2f55db21aa47419e60e05126aac4fe as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe 2024-11-24T02:44:57,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe, entries=150, sequenceid=441, filesize=12.0 K 2024-11-24T02:44:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/197f1a03d4e64021a06e794cf6d9eb10 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10 2024-11-24T02:44:57,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10, entries=150, sequenceid=441, filesize=12.0 K 2024-11-24T02:44:57,136 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for fa18a3ad7ebd64590be10753e6ed8f58 in 970ms, sequenceid=441, compaction requested=false 2024-11-24T02:44:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T02:44:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T02:44:57,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T02:44:57,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6930 sec 2024-11-24T02:44:57,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7030 sec 2024-11-24T02:44:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:57,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T02:44:57,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:57,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:57,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:57,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:57,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:57,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:44:57,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c4e1ec3a113f4b96940abfbb8da34079_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742449_1625 (size=17534) 2024-11-24T02:44:57,465 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:57,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c4e1ec3a113f4b96940abfbb8da34079_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c4e1ec3a113f4b96940abfbb8da34079_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:57,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3e8a29e4107c449281eb97b67b40938e, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:57,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3e8a29e4107c449281eb97b67b40938e is 175, key is test_row_0/A:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:57,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742450_1626 (size=48639) 2024-11-24T02:44:57,475 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3e8a29e4107c449281eb97b67b40938e 2024-11-24T02:44:57,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e54a1e7954974948a56a862bd80cd02c is 50, key is test_row_0/B:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:57,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742451_1627 (size=12301) 2024-11-24T02:44:57,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e54a1e7954974948a56a862bd80cd02c 2024-11-24T02:44:57,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/fed7ef87d1f14ce19771d9e78e6f91a3 is 50, key is test_row_0/C:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:57,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416357500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416357505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742452_1628 (size=12301) 2024-11-24T02:44:57,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416357618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416357619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52596 deadline: 1732416357829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:44:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34251 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52626 deadline: 1732416357829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., hostname=7c69a60bd8f6,34251,1732416122955, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:44:57,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/fed7ef87d1f14ce19771d9e78e6f91a3 2024-11-24T02:44:57,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/3e8a29e4107c449281eb97b67b40938e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e 2024-11-24T02:44:57,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e, entries=250, sequenceid=456, filesize=47.5 K 2024-11-24T02:44:57,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/e54a1e7954974948a56a862bd80cd02c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c 2024-11-24T02:44:57,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c, entries=150, sequenceid=456, filesize=12.0 K 2024-11-24T02:44:57,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/fed7ef87d1f14ce19771d9e78e6f91a3 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3 2024-11-24T02:44:57,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3, entries=150, sequenceid=456, filesize=12.0 K 2024-11-24T02:44:57,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fa18a3ad7ebd64590be10753e6ed8f58 in 492ms, sequenceid=456, compaction requested=true 2024-11-24T02:44:57,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:57,944 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:57,945 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fa18a3ad7ebd64590be10753e6ed8f58:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:44:57,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:57,945 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:57,945 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/A is initiating minor compaction (all files) 2024-11-24T02:44:57,945 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/A in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:57,945 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=109.3 K 2024-11-24T02:44:57,945 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/B is initiating minor compaction (all files) 2024-11-24T02:44:57,946 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:57,946 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/B in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. files: [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e] 2024-11-24T02:44:57,946 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17e772a0fef4756aa659808141305f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.8 K 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting a17e772a0fef4756aa659808141305f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab94dac511e4a788eae20fa4cb8c567, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c2f55db21aa47419e60e05126aac4fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732416294447 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452bc2f541674e37a15b6246a29ab22e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732416294447 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting e54a1e7954974948a56a862bd80cd02c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732416296301 2024-11-24T02:44:57,946 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8a29e4107c449281eb97b67b40938e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732416296301 2024-11-24T02:44:57,954 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#B#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:57,955 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ae33453242f74bf8ad3c76cb3b9c6931 is 50, key is test_row_0/B:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:57,961 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:57,993 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-24T02:44:57,993 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:57,993 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,993 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,993 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-24T02:44:57,993 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:57,993 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,994 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:57,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:57,995 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-24T02:44:57,995 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:57,995 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,995 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:57,996 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-24T02:44:57,996 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:57,996 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,996 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,996 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:57,997 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-24T02:44:57,997 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:57,997 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,997 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:57,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742453_1629 (size=13187) 2024-11-24T02:44:57,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:58,001 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e91fee9175094f3eac6556be65b04786_fa18a3ad7ebd64590be10753e6ed8f58 store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:58,003 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e91fee9175094f3eac6556be65b04786_fa18a3ad7ebd64590be10753e6ed8f58, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:58,003 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e91fee9175094f3eac6556be65b04786_fa18a3ad7ebd64590be10753e6ed8f58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:58,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742454_1630 (size=4469) 2024-11-24T02:44:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34251 {}] regionserver.HRegion(8855): Flush requested on fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:58,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-24T02:44:58,141 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-24T02:44:58,141 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:44:58,141 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:58,141 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:58,141 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,141 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,141 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,141 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:44:58,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:58,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T02:44:58,143 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:58,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240379d6e063584d8d8cf62f4fc95d9a69_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416298139/Put/seqid=0 2024-11-24T02:44:58,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742455_1631 (size=12454) 2024-11-24T02:44:58,402 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ae33453242f74bf8ad3c76cb3b9c6931 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ae33453242f74bf8ad3c76cb3b9c6931 2024-11-24T02:44:58,406 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/B of fa18a3ad7ebd64590be10753e6ed8f58 into ae33453242f74bf8ad3c76cb3b9c6931(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:58,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:58,406 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/B, priority=13, startTime=1732416297945; duration=0sec 2024-11-24T02:44:58,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:44:58,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:B 2024-11-24T02:44:58,406 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:44:58,407 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:44:58,407 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1541): fa18a3ad7ebd64590be10753e6ed8f58/C is initiating minor compaction (all files) 2024-11-24T02:44:58,407 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fa18a3ad7ebd64590be10753e6ed8f58/C in TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:44:58,407 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9be072c3abfb416f84868106c07ee10d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3] into tmpdir=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp, totalSize=36.8 K 2024-11-24T02:44:58,407 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be072c3abfb416f84868106c07ee10d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732416294245 2024-11-24T02:44:58,408 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting 197f1a03d4e64021a06e794cf6d9eb10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732416294447 2024-11-24T02:44:58,408 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] compactions.Compactor(225): Compacting fed7ef87d1f14ce19771d9e78e6f91a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732416296301 2024-11-24T02:44:58,411 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#A#compaction#529 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:58,412 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/386a3707fa1a4b99a037ee91cd35609a is 175, key is test_row_0/A:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:58,414 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fa18a3ad7ebd64590be10753e6ed8f58#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:44:58,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742456_1632 (size=32141) 2024-11-24T02:44:58,415 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/f230fe35dd7e4531af299eade85337ac is 50, key is test_row_0/C:col10/1732416296301/Put/seqid=0 2024-11-24T02:44:58,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742457_1633 (size=13187) 2024-11-24T02:44:58,419 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/386a3707fa1a4b99a037ee91cd35609a as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/386a3707fa1a4b99a037ee91cd35609a 2024-11-24T02:44:58,423 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/f230fe35dd7e4531af299eade85337ac as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f230fe35dd7e4531af299eade85337ac 2024-11-24T02:44:58,424 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/A of fa18a3ad7ebd64590be10753e6ed8f58 into 386a3707fa1a4b99a037ee91cd35609a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:58,424 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:58,424 INFO [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/A, priority=13, startTime=1732416297944; duration=0sec 2024-11-24T02:44:58,424 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:58,424 DEBUG [RS:0;7c69a60bd8f6:34251-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:A 2024-11-24T02:44:58,427 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fa18a3ad7ebd64590be10753e6ed8f58/C of fa18a3ad7ebd64590be10753e6ed8f58 into f230fe35dd7e4531af299eade85337ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:44:58,427 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:44:58,428 INFO [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., storeName=fa18a3ad7ebd64590be10753e6ed8f58/C, priority=13, startTime=1732416297945; duration=0sec 2024-11-24T02:44:58,428 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:44:58,428 DEBUG [RS:0;7c69a60bd8f6:34251-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fa18a3ad7ebd64590be10753e6ed8f58:C 2024-11-24T02:44:58,551 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:44:58,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240379d6e063584d8d8cf62f4fc95d9a69_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240379d6e063584d8d8cf62f4fc95d9a69_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:44:58,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4d51a42be89043748b8726d04b488f77, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:44:58,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4d51a42be89043748b8726d04b488f77 is 175, key is test_row_0/A:col10/1732416298139/Put/seqid=0 2024-11-24T02:44:58,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742458_1634 (size=31255) 2024-11-24T02:44:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:44:58,599 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:44:58,650 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-24T02:44:58,650 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:58,650 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,650 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:58,660 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-24T02:44:58,660 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:44:58,661 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,661 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:44:58,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:44:58,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4d51a42be89043748b8726d04b488f77 2024-11-24T02:44:58,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/58ac7814870048db99431dd67dad09cf is 50, key is test_row_0/B:col10/1732416298139/Put/seqid=0 2024-11-24T02:44:58,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742459_1635 (size=12301) 2024-11-24T02:44:59,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/58ac7814870048db99431dd67dad09cf 2024-11-24T02:44:59,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3175060acbe445a9afda3ad72bbf21b0 is 50, key is test_row_0/C:col10/1732416298139/Put/seqid=0 2024-11-24T02:44:59,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742460_1636 (size=12301) 2024-11-24T02:44:59,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3175060acbe445a9afda3ad72bbf21b0 2024-11-24T02:44:59,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/4d51a42be89043748b8726d04b488f77 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4d51a42be89043748b8726d04b488f77 2024-11-24T02:44:59,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4d51a42be89043748b8726d04b488f77, entries=150, sequenceid=478, filesize=30.5 K 2024-11-24T02:44:59,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/58ac7814870048db99431dd67dad09cf as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/58ac7814870048db99431dd67dad09cf 2024-11-24T02:44:59,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/58ac7814870048db99431dd67dad09cf, entries=150, sequenceid=478, filesize=12.0 K 2024-11-24T02:44:59,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/3175060acbe445a9afda3ad72bbf21b0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3175060acbe445a9afda3ad72bbf21b0 2024-11-24T02:44:59,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3175060acbe445a9afda3ad72bbf21b0, entries=150, sequenceid=478, filesize=12.0 K 2024-11-24T02:44:59,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=13.42 KB/13740 for fa18a3ad7ebd64590be10753e6ed8f58 in 1655ms, sequenceid=478, compaction requested=false 2024-11-24T02:44:59,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fa18a3ad7ebd64590be10753e6ed8f58: 2024-11-24T02:45:01,290 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:45:06,373 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-24T02:45:06,373 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:45:06,373 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:06,374 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:06,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2756 2024-11-24T02:45:06,374 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2740 2024-11-24T02:45:06,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2735 2024-11-24T02:45:06,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2740 2024-11-24T02:45:06,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2730 2024-11-24T02:45:06,375 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:45:06,375 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:45:06,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:45:06,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:06,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:06,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:45:06,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:45:06,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:06,382 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416306382"}]},"ts":"1732416306382"} 2024-11-24T02:45:06,384 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:45:06,384 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:45:06,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:45:06,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, UNASSIGN}] 2024-11-24T02:45:06,388 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, UNASSIGN 2024-11-24T02:45:06,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=CLOSING, regionLocation=7c69a60bd8f6,34251,1732416122955 2024-11-24T02:45:06,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, UNASSIGN because future has completed 2024-11-24T02:45:06,390 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:45:06,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955}] 2024-11-24T02:45:06,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:06,543 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:06,543 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:45:06,543 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing fa18a3ad7ebd64590be10753e6ed8f58, disabling compactions & flushes 2024-11-24T02:45:06,544 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. after waiting 0 ms 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:45:06,544 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing fa18a3ad7ebd64590be10753e6ed8f58 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=A 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=B 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fa18a3ad7ebd64590be10753e6ed8f58, store=C 2024-11-24T02:45:06,544 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T02:45:06,548 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ad8ff7e71dc745c3ae968c91841d3d83_fa18a3ad7ebd64590be10753e6ed8f58 is 50, key is test_row_0/A:col10/1732416306370/Put/seqid=0 2024-11-24T02:45:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742461_1637 (size=9914) 2024-11-24T02:45:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:06,951 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:45:06,955 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ad8ff7e71dc745c3ae968c91841d3d83_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad8ff7e71dc745c3ae968c91841d3d83_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:06,956 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/c323aef55de049528f53beb933173ed7, store: [table=TestAcidGuarantees family=A region=fa18a3ad7ebd64590be10753e6ed8f58] 2024-11-24T02:45:06,957 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/c323aef55de049528f53beb933173ed7 is 175, key is test_row_0/A:col10/1732416306370/Put/seqid=0 2024-11-24T02:45:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742462_1638 (size=22561) 2024-11-24T02:45:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:07,361 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=487, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/c323aef55de049528f53beb933173ed7 2024-11-24T02:45:07,367 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ba1cab9c86994de09350a7adafbe6bd0 is 50, key is test_row_0/B:col10/1732416306370/Put/seqid=0 2024-11-24T02:45:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742463_1639 (size=9857) 2024-11-24T02:45:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:07,770 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ba1cab9c86994de09350a7adafbe6bd0 2024-11-24T02:45:07,777 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1e66987f6ea646df8577ee4af0bc4029 is 50, key is test_row_0/C:col10/1732416306370/Put/seqid=0 2024-11-24T02:45:07,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742464_1640 (size=9857) 2024-11-24T02:45:08,181 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1e66987f6ea646df8577ee4af0bc4029 2024-11-24T02:45:08,185 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/A/c323aef55de049528f53beb933173ed7 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/c323aef55de049528f53beb933173ed7 2024-11-24T02:45:08,189 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/c323aef55de049528f53beb933173ed7, entries=100, sequenceid=487, filesize=22.0 K 2024-11-24T02:45:08,189 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/B/ba1cab9c86994de09350a7adafbe6bd0 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ba1cab9c86994de09350a7adafbe6bd0 2024-11-24T02:45:08,192 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ba1cab9c86994de09350a7adafbe6bd0, entries=100, sequenceid=487, filesize=9.6 K 2024-11-24T02:45:08,193 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/.tmp/C/1e66987f6ea646df8577ee4af0bc4029 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1e66987f6ea646df8577ee4af0bc4029 2024-11-24T02:45:08,195 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1e66987f6ea646df8577ee4af0bc4029, entries=100, sequenceid=487, filesize=9.6 K 2024-11-24T02:45:08,196 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fa18a3ad7ebd64590be10753e6ed8f58 in 1652ms, sequenceid=487, compaction requested=true 2024-11-24T02:45:08,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e] to archive 2024-11-24T02:45:08,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:45:08,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/cbf823c07e3d4439b56ba3bc9b33eabf 2024-11-24T02:45:08,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/dd7e3336ab2a4413ace40fde0e82f8b8 2024-11-24T02:45:08,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f5ccd6b4ecdb4fbda12f8d4de02b5518 2024-11-24T02:45:08,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb3f3f44ffe4121a1142b9c351fd007 2024-11-24T02:45:08,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4c3355e4c7614d4b9e40708d96c2d2ec 2024-11-24T02:45:08,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/69faa44c1b224aecb240c18e8fc96bc9 2024-11-24T02:45:08,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f0f22a0c28554f88babf7707a815cbfd 2024-11-24T02:45:08,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/27d4e488231949199fa4fe3efd69ea12 2024-11-24T02:45:08,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/94b9f57531504fe2ad3a45d981cbf18f 2024-11-24T02:45:08,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6f7a1fb7581649729c548dd1b135f950 2024-11-24T02:45:08,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6b63026df10d444f8c140c0b1a0d9e59 2024-11-24T02:45:08,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/2fb1114a1d45411e80aa1bb72b63c3c5 2024-11-24T02:45:08,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/f400a5f1b7e847db96d0c53846b49a47 2024-11-24T02:45:08,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7eb5e6d374524b95ae101bef0bb45a71 2024-11-24T02:45:08,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/7171bf64f55c486f8b4db6469dabfbba 2024-11-24T02:45:08,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3b530ed64d29431a9392016b033a93fc 2024-11-24T02:45:08,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/8c595f6ea32a4661805108965221ccca 2024-11-24T02:45:08,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/a83d3e58bc5543d080980c2e0ddce0ce 2024-11-24T02:45:08,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/9332750baed64529b7db8f9d5cc4809b 2024-11-24T02:45:08,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/6ce87c5887fe4de0bdca01edd31cc1d6 2024-11-24T02:45:08,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/1479bda27b4c4c10be6787142797824f 2024-11-24T02:45:08,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/d2aeaa4b9ec44470814aae2ef8b5cd61 2024-11-24T02:45:08,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/bab94dac511e4a788eae20fa4cb8c567 2024-11-24T02:45:08,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/45c9a13598b3400e864ad91523170ca1 2024-11-24T02:45:08,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/452bc2f541674e37a15b6246a29ab22e 2024-11-24T02:45:08,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/3e8a29e4107c449281eb97b67b40938e 2024-11-24T02:45:08,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/b5ed71a4fbae4fd4ac0f527227effcaa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/0e6e7a7ed3c34729873ccdc45c7ec61a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17caffed45941bb85c8173d132a218a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4451d629e0524fa480e85c78dcb0e61e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/1bc93c6f97b740f0930aa7b61ee88021, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f06735f8a52f4814b9e4d21b09536795, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17e772a0fef4756aa659808141305f1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c] to archive 2024-11-24T02:45:08,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:45:08,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/3dc453f10a58460c95aa128528c36a5a 2024-11-24T02:45:08,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e428dfd49cff45928c8a83333d9ed47a 2024-11-24T02:45:08,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/b5ed71a4fbae4fd4ac0f527227effcaa to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/b5ed71a4fbae4fd4ac0f527227effcaa 2024-11-24T02:45:08,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/022b4d141e984311990fe61ba7b8b19f 2024-11-24T02:45:08,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/67266ac335cc488a8014a21a052628c3 2024-11-24T02:45:08,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/497901615aec48e0a80945b95e36efc6 2024-11-24T02:45:08,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/0e6e7a7ed3c34729873ccdc45c7ec61a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/0e6e7a7ed3c34729873ccdc45c7ec61a 2024-11-24T02:45:08,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/090b3f2700e341d79728cdd4787fccbd 2024-11-24T02:45:08,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4cde6ba21cd64118bb7ba253958fd380 2024-11-24T02:45:08,226 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17caffed45941bb85c8173d132a218a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17caffed45941bb85c8173d132a218a 2024-11-24T02:45:08,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ea3d7d4d610a45b0a2fa2af2b52d54de 2024-11-24T02:45:08,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/dfe5d57b5fdd4fae96600657c8cd648f 2024-11-24T02:45:08,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4451d629e0524fa480e85c78dcb0e61e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4451d629e0524fa480e85c78dcb0e61e 2024-11-24T02:45:08,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/70be2bb67aca4c65a9b63cc14aecdda5 2024-11-24T02:45:08,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f0adf050f5ac471baf88ae7465728b97 2024-11-24T02:45:08,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/6e4ca9a3cfd5492bb191a87b9c3f9f66 2024-11-24T02:45:08,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/1bc93c6f97b740f0930aa7b61ee88021 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/1bc93c6f97b740f0930aa7b61ee88021 2024-11-24T02:45:08,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/fc646943a75a41629003d1f8a295c312 2024-11-24T02:45:08,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/4429c816b7304154bb7f404679c8af4e 2024-11-24T02:45:08,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f06735f8a52f4814b9e4d21b09536795 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/f06735f8a52f4814b9e4d21b09536795 2024-11-24T02:45:08,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/c43228cb92354e03bc3ac040995fbee6 2024-11-24T02:45:08,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/15b619f1bb5c4c5cb21153d0b52bdff1 2024-11-24T02:45:08,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17e772a0fef4756aa659808141305f1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/a17e772a0fef4756aa659808141305f1 2024-11-24T02:45:08,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/da6421374ca64ab7bb9b78df7a65b3ce 2024-11-24T02:45:08,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/8c2f55db21aa47419e60e05126aac4fe 2024-11-24T02:45:08,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/e54a1e7954974948a56a862bd80cd02c 2024-11-24T02:45:08,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a2e4126970479aaf23d5a2f1b80172, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3cc544561bf04191bb67b4aacda69d4c, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5cf1cc993a134285b69db89c5428331e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/6b02ededb4534349b49a280106d2a8a8, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d89e0eed0b944e5fa35d9e657b900d29, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3041a15095de4f2daa11538f54abfd6a, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9be072c3abfb416f84868106c07ee10d, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3] to archive 2024-11-24T02:45:08,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:45:08,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d26409ac69f94edba349648058742474 2024-11-24T02:45:08,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a23cbb04f14e1799c0aa928e756c72 2024-11-24T02:45:08,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a2e4126970479aaf23d5a2f1b80172 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/a2a2e4126970479aaf23d5a2f1b80172 2024-11-24T02:45:08,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9d28daaeadf14708b8bc5bedee6af2b9 2024-11-24T02:45:08,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5fad8e73fb0b495bb1d979f0976faa0b 2024-11-24T02:45:08,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/4ac41096444a4fb6bde6a822fa1f4108 2024-11-24T02:45:08,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3cc544561bf04191bb67b4aacda69d4c to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3cc544561bf04191bb67b4aacda69d4c 2024-11-24T02:45:08,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/7f29459b230a408f9ce687717d96c893 2024-11-24T02:45:08,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/c0b4833d5b5c40b087fea47603739222 2024-11-24T02:45:08,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5cf1cc993a134285b69db89c5428331e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5cf1cc993a134285b69db89c5428331e 2024-11-24T02:45:08,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9448bfde47094fa7bdfb9727de1786d8 2024-11-24T02:45:08,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/86165ff8b7e6470da334cc001c4269aa 2024-11-24T02:45:08,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/6b02ededb4534349b49a280106d2a8a8 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/6b02ededb4534349b49a280106d2a8a8 2024-11-24T02:45:08,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/633120ef79b14e7699c421955922d10f 2024-11-24T02:45:08,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1d450e3af6ff4339ab9a14a5d04ffbb1 2024-11-24T02:45:08,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9bb0b7725dcb489db30f971bc37274ae 2024-11-24T02:45:08,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d89e0eed0b944e5fa35d9e657b900d29 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/d89e0eed0b944e5fa35d9e657b900d29 2024-11-24T02:45:08,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/77d8f721c88048308138d35eef1c6a2e 2024-11-24T02:45:08,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/553e7e9609bc49d4a9a9f98cae750e40 2024-11-24T02:45:08,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3041a15095de4f2daa11538f54abfd6a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3041a15095de4f2daa11538f54abfd6a 2024-11-24T02:45:08,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3591b2ca008b47a0a6d3e1eb71f21b51 2024-11-24T02:45:08,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f11d180736764d27aa3c0d9ecf3395bb 2024-11-24T02:45:08,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9be072c3abfb416f84868106c07ee10d to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/9be072c3abfb416f84868106c07ee10d 2024-11-24T02:45:08,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/5b2ddc348d3c457a9b371a0a35b11436 2024-11-24T02:45:08,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/197f1a03d4e64021a06e794cf6d9eb10 2024-11-24T02:45:08,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/fed7ef87d1f14ce19771d9e78e6f91a3 2024-11-24T02:45:08,280 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits/490.seqid, newMaxSeqId=490, maxSeqId=4 2024-11-24T02:45:08,280 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. 2024-11-24T02:45:08,281 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for fa18a3ad7ebd64590be10753e6ed8f58: Waiting for close lock at 1732416306543Running coprocessor pre-close hooks at 1732416306543Disabling compacts and flushes for region at 1732416306543Disabling writes for close at 1732416306544 (+1 ms)Obtaining lock to block concurrent updates at 1732416306544Preparing flush snapshotting stores in fa18a3ad7ebd64590be10753e6ed8f58 at 1732416306544Finished memstore snapshotting TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732416306544Flushing stores of TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58. at 1732416306544Flushing fa18a3ad7ebd64590be10753e6ed8f58/A: creating writer at 1732416306544Flushing mob file fa18a3ad7ebd64590be10753e6ed8f58/A: appending metadata at 1732416306548 (+4 ms)Flushing mob file fa18a3ad7ebd64590be10753e6ed8f58/A: closing flushed file at 1732416306548Flushing fa18a3ad7ebd64590be10753e6ed8f58/A: appending metadata at 1732416306956 (+408 ms)Flushing fa18a3ad7ebd64590be10753e6ed8f58/A: closing flushed file at 1732416306956Flushing fa18a3ad7ebd64590be10753e6ed8f58/B: creating writer at 1732416307364 (+408 ms)Flushing fa18a3ad7ebd64590be10753e6ed8f58/B: appending metadata at 1732416307367 (+3 ms)Flushing fa18a3ad7ebd64590be10753e6ed8f58/B: closing flushed file at 1732416307367Flushing fa18a3ad7ebd64590be10753e6ed8f58/C: creating writer at 1732416307774 (+407 ms)Flushing fa18a3ad7ebd64590be10753e6ed8f58/C: appending metadata at 1732416307776 (+2 ms)Flushing fa18a3ad7ebd64590be10753e6ed8f58/C: closing flushed file at 1732416307776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54405c48: reopening flushed file at 1732416308184 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b3181b0: reopening flushed file at 1732416308189 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7263d499: reopening flushed file at 1732416308192 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for fa18a3ad7ebd64590be10753e6ed8f58 in 1652ms, sequenceid=487, compaction requested=true at 1732416308196 (+4 ms)Writing region close event to WAL at 1732416308278 (+82 ms)Running coprocessor post-close hooks at 1732416308280 (+2 ms)Closed at 1732416308280 2024-11-24T02:45:08,282 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,282 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fa18a3ad7ebd64590be10753e6ed8f58, regionState=CLOSED 2024-11-24T02:45:08,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 because future has completed 2024-11-24T02:45:08,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T02:45:08,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure fa18a3ad7ebd64590be10753e6ed8f58, server=7c69a60bd8f6,34251,1732416122955 in 1.8940 sec 2024-11-24T02:45:08,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-24T02:45:08,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fa18a3ad7ebd64590be10753e6ed8f58, UNASSIGN in 1.9000 sec 2024-11-24T02:45:08,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T02:45:08,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9020 sec 2024-11-24T02:45:08,289 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416308289"}]},"ts":"1732416308289"} 2024-11-24T02:45:08,290 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:45:08,290 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:45:08,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-24T02:45:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:45:08,529 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:45:08,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:45:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,531 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:45:08,532 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,533 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,535 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C, FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits] 2024-11-24T02:45:08,536 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/386a3707fa1a4b99a037ee91cd35609a to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/386a3707fa1a4b99a037ee91cd35609a 2024-11-24T02:45:08,537 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4d51a42be89043748b8726d04b488f77 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/4d51a42be89043748b8726d04b488f77 2024-11-24T02:45:08,538 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/c323aef55de049528f53beb933173ed7 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/A/c323aef55de049528f53beb933173ed7 2024-11-24T02:45:08,539 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/58ac7814870048db99431dd67dad09cf to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/58ac7814870048db99431dd67dad09cf 2024-11-24T02:45:08,540 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ae33453242f74bf8ad3c76cb3b9c6931 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ae33453242f74bf8ad3c76cb3b9c6931 2024-11-24T02:45:08,540 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ba1cab9c86994de09350a7adafbe6bd0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/B/ba1cab9c86994de09350a7adafbe6bd0 2024-11-24T02:45:08,542 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1e66987f6ea646df8577ee4af0bc4029 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/1e66987f6ea646df8577ee4af0bc4029 2024-11-24T02:45:08,542 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3175060acbe445a9afda3ad72bbf21b0 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/3175060acbe445a9afda3ad72bbf21b0 2024-11-24T02:45:08,543 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f230fe35dd7e4531af299eade85337ac to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/C/f230fe35dd7e4531af299eade85337ac 2024-11-24T02:45:08,544 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits/490.seqid to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58/recovered.edits/490.seqid 2024-11-24T02:45:08,545 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/default/TestAcidGuarantees/fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:45:08,545 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:45:08,545 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:45:08,547 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124004dc4aa8ad143a9a631ec1f1bdf714e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124004dc4aa8ad143a9a631ec1f1bdf714e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,548 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240379d6e063584d8d8cf62f4fc95d9a69_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240379d6e063584d8d8cf62f4fc95d9a69_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,549 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242252fdfeaa9c4cf69b88c645ac66290b_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242252fdfeaa9c4cf69b88c645ac66290b_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,550 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ca31e6d284c4c6bbf59fb963b937c05_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242ca31e6d284c4c6bbf59fb963b937c05_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,551 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f521c4a689b4b85872fe5e1ccde090e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242f521c4a689b4b85872fe5e1ccde090e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,552 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243061b225b36a4f2c8103d824c051591e_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243061b225b36a4f2c8103d824c051591e_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,553 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243717e995eb984fd68511065fa48a4557_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243717e995eb984fd68511065fa48a4557_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,554 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b77d466170c49f28f507d6880a00681_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b77d466170c49f28f507d6880a00681_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,555 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451d9241d3b244aa999eea3d4d18ee8ba_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451d9241d3b244aa999eea3d4d18ee8ba_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,557 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458ba3fc8dc504635b0993183ef1fe5a4_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112458ba3fc8dc504635b0993183ef1fe5a4_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,558 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246404e5b7d6a04a479ef0f4ed60157baf_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246404e5b7d6a04a479ef0f4ed60157baf_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,559 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248932e1697772497e9f7b3288c35df7b0_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248932e1697772497e9f7b3288c35df7b0_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,560 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248cef208ea95d4c838bc87cc5d407d641_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248cef208ea95d4c838bc87cc5d407d641_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,561 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a104291dfa5a4cf29ef95845eb236b76_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a104291dfa5a4cf29ef95845eb236b76_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,562 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad8ff7e71dc745c3ae968c91841d3d83_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ad8ff7e71dc745c3ae968c91841d3d83_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,563 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2384e3914bf4159a63626de9ff10be9_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2384e3914bf4159a63626de9ff10be9_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,564 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c4e1ec3a113f4b96940abfbb8da34079_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c4e1ec3a113f4b96940abfbb8da34079_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,566 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f5521909c14303b8e4731b001ddc41_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5f5521909c14303b8e4731b001ddc41_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,567 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c8736417fee94f2e98c27b8424904e79_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c8736417fee94f2e98c27b8424904e79_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,569 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df3b6ccfcb6b4979bd93ba2c41c49717_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124df3b6ccfcb6b4979bd93ba2c41c49717_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,570 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f4ba564af24742fc8f966f60a895bfcc_fa18a3ad7ebd64590be10753e6ed8f58 to hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f4ba564af24742fc8f966f60a895bfcc_fa18a3ad7ebd64590be10753e6ed8f58 2024-11-24T02:45:08,571 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:45:08,574 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,577 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:45:08,579 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:45:08,580 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,580 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:45:08,580 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416308580"}]},"ts":"9223372036854775807"} 2024-11-24T02:45:08,581 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:45:08,581 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fa18a3ad7ebd64590be10753e6ed8f58, NAME => 'TestAcidGuarantees,,1732416274855.fa18a3ad7ebd64590be10753e6ed8f58.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:45:08,582 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:45:08,582 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416308582"}]},"ts":"9223372036854775807"} 2024-11-24T02:45:08,583 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:45:08,583 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:45:08,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-24T02:45:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:45:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:45:08,643 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:45:08,653 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=445 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=471 (was 347) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9764 (was 10061) 2024-11-24T02:45:08,653 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T02:45:08,653 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:45:08,653 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:45:08,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:08,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:08,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:45:08,653 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T02:45:08,654 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1484730631, stopped=false 2024-11-24T02:45:08,654 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c69a60bd8f6,34019,1732416122189 2024-11-24T02:45:08,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T02:45:08,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T02:45:08,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:45:08,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:45:08,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:45:08,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:45:08,667 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:45:08,667 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:45:08,667 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:45:08,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:08,667 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c69a60bd8f6,34251,1732416122955' ***** 2024-11-24T02:45:08,667 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T02:45:08,668 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(959): stopping server 7c69a60bd8f6,34251,1732416122955 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c69a60bd8f6:34251. 2024-11-24T02:45:08,668 DEBUG [RS:0;7c69a60bd8f6:34251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:45:08,668 DEBUG [RS:0;7c69a60bd8f6:34251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:08,668 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T02:45:08,669 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T02:45:08,669 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T02:45:08,669 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T02:45:08,669 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T02:45:08,669 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T02:45:08,669 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:08,669 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:45:08,669 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:45:08,669 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:45:08,670 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:45:08,670 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:45:08,670 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T02:45:08,690 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/info/8db77dc75f1149829214c10b40dc066e is 95, key is TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e./info:/1732416154534/DeleteFamily/seqid=0 2024-11-24T02:45:08,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742465_1641 (size=6514) 2024-11-24T02:45:08,733 INFO [regionserver/7c69a60bd8f6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:45:08,870 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:08,935 INFO [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T02:45:08,935 INFO [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T02:45:09,070 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:09,094 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/info/8db77dc75f1149829214c10b40dc066e 2024-11-24T02:45:09,113 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/ns/c90c0e043c984721acac9b5374aef871 is 93, key is TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e./ns:/1732416154475/DeleteFamily/seqid=0 2024-11-24T02:45:09,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742466_1642 (size=5906) 2024-11-24T02:45:09,270 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:09,470 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:09,519 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/ns/c90c0e043c984721acac9b5374aef871 2024-11-24T02:45:09,543 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/rep_barrier/92eb7adb03e5450583de096532187b4c is 102, key is TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e./rep_barrier:/1732416154475/DeleteFamily/seqid=0 2024-11-24T02:45:09,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742467_1643 (size=6025) 2024-11-24T02:45:09,670 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:09,871 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:09,949 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/rep_barrier/92eb7adb03e5450583de096532187b4c 2024-11-24T02:45:09,967 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/table/ca740508271645e5803f61f39cd88e25 is 96, key is TestAcidGuarantees,,1732416127577.94fe1d236a5fa4edae1b3d10d452f90e./table:/1732416154475/DeleteFamily/seqid=0 2024-11-24T02:45:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742468_1644 (size=5936) 2024-11-24T02:45:10,071 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:10,271 DEBUG [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:45:10,371 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/table/ca740508271645e5803f61f39cd88e25 2024-11-24T02:45:10,374 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/info/8db77dc75f1149829214c10b40dc066e as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/info/8db77dc75f1149829214c10b40dc066e 2024-11-24T02:45:10,377 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/info/8db77dc75f1149829214c10b40dc066e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T02:45:10,378 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/ns/c90c0e043c984721acac9b5374aef871 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/ns/c90c0e043c984721acac9b5374aef871 2024-11-24T02:45:10,381 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/ns/c90c0e043c984721acac9b5374aef871, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T02:45:10,382 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/rep_barrier/92eb7adb03e5450583de096532187b4c as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/rep_barrier/92eb7adb03e5450583de096532187b4c 2024-11-24T02:45:10,385 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/rep_barrier/92eb7adb03e5450583de096532187b4c, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T02:45:10,386 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/.tmp/table/ca740508271645e5803f61f39cd88e25 as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/table/ca740508271645e5803f61f39cd88e25 2024-11-24T02:45:10,389 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/table/ca740508271645e5803f61f39cd88e25, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T02:45:10,390 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false 2024-11-24T02:45:10,394 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T02:45:10,395 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:45:10,395 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:45:10,395 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732416308669Running coprocessor pre-close hooks at 1732416308669Disabling compacts and flushes for region at 1732416308669Disabling writes for close at 1732416308670 (+1 ms)Obtaining lock to block concurrent updates at 1732416308670Preparing flush snapshotting stores in 1588230740 at 1732416308670Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732416308670Flushing stores of hbase:meta,,1.1588230740 at 1732416308670Flushing 1588230740/info: creating writer at 1732416308671 (+1 ms)Flushing 1588230740/info: appending metadata at 1732416308690 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732416308690Flushing 1588230740/ns: creating writer at 1732416309097 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732416309112 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732416309112Flushing 1588230740/rep_barrier: creating writer at 1732416309524 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732416309542 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732416309542Flushing 1588230740/table: creating writer at 1732416309953 (+411 ms)Flushing 1588230740/table: appending metadata at 1732416309967 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732416309967Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f529206: reopening flushed file at 1732416310374 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68f8e66b: reopening flushed file at 1732416310378 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7380bba4: reopening flushed file at 1732416310382 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e0b4cc: reopening flushed file at 1732416310385 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false at 1732416310390 (+5 ms)Writing region close event to WAL at 1732416310392 (+2 ms)Running coprocessor post-close hooks at 1732416310395 (+3 ms)Closed at 1732416310395 2024-11-24T02:45:10,395 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T02:45:10,471 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(976): stopping server 7c69a60bd8f6,34251,1732416122955; all regions closed. 2024-11-24T02:45:10,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741834_1010 (size=25344) 2024-11-24T02:45:10,479 DEBUG [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/oldWALs 2024-11-24T02:45:10,479 INFO [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7c69a60bd8f6%2C34251%2C1732416122955.meta:.meta(num 1732416127086) 2024-11-24T02:45:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741832_1008 (size=15944682) 2024-11-24T02:45:10,483 DEBUG [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/oldWALs 2024-11-24T02:45:10,483 INFO [RS:0;7c69a60bd8f6:34251 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7c69a60bd8f6%2C34251%2C1732416122955:(num 1732416126088) 2024-11-24T02:45:10,483 DEBUG [RS:0;7c69a60bd8f6:34251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:45:10,483 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:45:10,483 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:45:10,484 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.ChoreService(370): Chore service for: regionserver/7c69a60bd8f6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T02:45:10,484 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:45:10,484 INFO [regionserver/7c69a60bd8f6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:45:10,485 INFO [RS:0;7c69a60bd8f6:34251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34251 2024-11-24T02:45:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c69a60bd8f6,34251,1732416122955 2024-11-24T02:45:10,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T02:45:10,498 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:45:10,499 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c69a60bd8f6,34251,1732416122955] 2024-11-24T02:45:10,518 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c69a60bd8f6,34251,1732416122955 already deleted, retry=false 2024-11-24T02:45:10,519 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c69a60bd8f6,34251,1732416122955 expired; onlineServers=0 2024-11-24T02:45:10,519 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c69a60bd8f6,34019,1732416122189' ***** 2024-11-24T02:45:10,519 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T02:45:10,519 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:45:10,519 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:45:10,519 DEBUG [M:0;7c69a60bd8f6:34019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T02:45:10,520 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T02:45:10,520 DEBUG [M:0;7c69a60bd8f6:34019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T02:45:10,520 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416125758 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416125758,5,FailOnTimeoutGroup] 2024-11-24T02:45:10,520 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.ChoreService(370): Chore service for: master/7c69a60bd8f6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T02:45:10,520 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416125759 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416125759,5,FailOnTimeoutGroup] 2024-11-24T02:45:10,520 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:45:10,520 DEBUG [M:0;7c69a60bd8f6:34019 {}] master.HMaster(1795): Stopping service threads 2024-11-24T02:45:10,520 INFO [M:0;7c69a60bd8f6:34019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T02:45:10,520 INFO [M:0;7c69a60bd8f6:34019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:45:10,520 ERROR [M:0;7c69a60bd8f6:34019 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T02:45:10,521 INFO [M:0;7c69a60bd8f6:34019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T02:45:10,521 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T02:45:10,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T02:45:10,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:45:10,529 DEBUG [M:0;7c69a60bd8f6:34019 {}] zookeeper.ZKUtil(347): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T02:45:10,530 WARN [M:0;7c69a60bd8f6:34019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T02:45:10,530 INFO [M:0;7c69a60bd8f6:34019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/.lastflushedseqids 2024-11-24T02:45:10,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742469_1645 (size=72) 2024-11-24T02:45:10,540 INFO [M:0;7c69a60bd8f6:34019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T02:45:10,540 INFO [M:0;7c69a60bd8f6:34019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T02:45:10,541 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:45:10,541 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:45:10,541 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:45:10,541 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:45:10,541 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:45:10,541 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=763.56 KB heapSize=934.93 KB 2024-11-24T02:45:10,559 DEBUG [M:0;7c69a60bd8f6:34019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08e986d793f443dd8e19b670d480771d is 82, key is hbase:meta,,1/info:regioninfo/1732416127166/Put/seqid=0 2024-11-24T02:45:10,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742470_1646 (size=5672) 2024-11-24T02:45:10,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:45:10,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1016ab825bb0001, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:45:10,609 INFO [RS:0;7c69a60bd8f6:34251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:45:10,609 INFO [RS:0;7c69a60bd8f6:34251 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c69a60bd8f6,34251,1732416122955; zookeeper connection closed. 2024-11-24T02:45:10,610 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ec43cf8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ec43cf8 2024-11-24T02:45:10,610 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T02:45:10,965 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08e986d793f443dd8e19b670d480771d 2024-11-24T02:45:10,996 DEBUG [M:0;7c69a60bd8f6:34019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc0dce4a9f94804b81113e285bb8cab is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1732416277953/Put/seqid=0 2024-11-24T02:45:11,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742471_1647 (size=45008) 2024-11-24T02:45:11,000 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.00 KB at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc0dce4a9f94804b81113e285bb8cab 2024-11-24T02:45:11,004 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0dc0dce4a9f94804b81113e285bb8cab 2024-11-24T02:45:11,030 DEBUG [M:0;7c69a60bd8f6:34019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c03e41118b304f60a6e8818c16aca2ba is 69, key is 7c69a60bd8f6,34251,1732416122955/rs:state/1732416125848/Put/seqid=0 2024-11-24T02:45:11,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073742472_1648 (size=5156) 2024-11-24T02:45:11,037 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c03e41118b304f60a6e8818c16aca2ba 2024-11-24T02:45:11,041 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/08e986d793f443dd8e19b670d480771d as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/08e986d793f443dd8e19b670d480771d 2024-11-24T02:45:11,044 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/08e986d793f443dd8e19b670d480771d, entries=8, sequenceid=2111, filesize=5.5 K 2024-11-24T02:45:11,045 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0dc0dce4a9f94804b81113e285bb8cab as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dc0dce4a9f94804b81113e285bb8cab 2024-11-24T02:45:11,048 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0dc0dce4a9f94804b81113e285bb8cab 2024-11-24T02:45:11,048 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0dc0dce4a9f94804b81113e285bb8cab, entries=168, sequenceid=2111, filesize=44.0 K 2024-11-24T02:45:11,048 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c03e41118b304f60a6e8818c16aca2ba as hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c03e41118b304f60a6e8818c16aca2ba 2024-11-24T02:45:11,052 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41729/user/jenkins/test-data/806c2019-8efd-5983-171a-b234bcae64c6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c03e41118b304f60a6e8818c16aca2ba, entries=1, sequenceid=2111, filesize=5.0 K 2024-11-24T02:45:11,053 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~763.56 KB/781884, heapSize ~934.63 KB/957064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 512ms, sequenceid=2111, compaction requested=false 2024-11-24T02:45:11,076 INFO [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:45:11,077 DEBUG [M:0;7c69a60bd8f6:34019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732416310541Disabling compacts and flushes for region at 1732416310541Disabling writes for close at 1732416310541Obtaining lock to block concurrent updates at 1732416310541Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732416310541Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=781884, getHeapSize=957304, getOffHeapSize=0, getCellsCount=2304 at 1732416310541Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732416310542 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732416310542Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732416310559 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732416310559Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732416310969 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732416310995 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732416310995Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732416311005 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732416311029 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732416311029Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa9f377: reopening flushed file at 1732416311041 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a44c7bc: reopening flushed file at 1732416311044 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@379e2d92: reopening flushed file at 1732416311048 (+4 ms)Finished flush of dataSize ~763.56 KB/781884, heapSize ~934.63 KB/957064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 512ms, sequenceid=2111, compaction requested=false at 1732416311053 (+5 ms)Writing region close event to WAL at 1732416311076 (+23 ms)Closed at 1732416311076 2024-11-24T02:45:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46637 is added to blk_1073741830_1006 (size=920290) 2024-11-24T02:45:11,082 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:45:11,082 INFO [M:0;7c69a60bd8f6:34019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T02:45:11,082 INFO [M:0;7c69a60bd8f6:34019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34019 2024-11-24T02:45:11,082 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:45:11,240 INFO [M:0;7c69a60bd8f6:34019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:45:11,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:45:11,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34019-0x1016ab825bb0000, quorum=127.0.0.1:59990, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:45:11,245 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:45:11,248 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:45:11,248 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:45:11,248 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:45:11,249 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.log.dir/,STOPPED} 2024-11-24T02:45:11,253 WARN [BP-1872727932-172.17.0.2-1732416118302 heartbeating to localhost/127.0.0.1:41729 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T02:45:11,253 WARN [BP-1872727932-172.17.0.2-1732416118302 heartbeating to localhost/127.0.0.1:41729 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1872727932-172.17.0.2-1732416118302 (Datanode Uuid d0a38681-d74f-4f7b-a7d8-12c8611fc8b0) service to localhost/127.0.0.1:41729 2024-11-24T02:45:11,255 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T02:45:11,255 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T02:45:11,256 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/data/data1/current/BP-1872727932-172.17.0.2-1732416118302 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:45:11,257 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/cluster_34396e74-36bf-5267-e66f-cc9bf9edb6e3/data/data2/current/BP-1872727932-172.17.0.2-1732416118302 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:45:11,257 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T02:45:11,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:45:11,267 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:45:11,267 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:45:11,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:45:11,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fe56365c-083c-a282-597d-35c34982c0cb/hadoop.log.dir/,STOPPED} 2024-11-24T02:45:11,287 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T02:45:11,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down